fix: create parent dirs in script new and accept python as language alias

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/benchmark.yml

@@ -290,49 +290,6 @@ jobs:
           path: |
             *.json
 
-  benchmark_wac:
-    runs-on: ubicloud-standard-8
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: windmill
-          POSTGRES_PASSWORD: changeme
-          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
-        options: >-
-          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
-          --health-retries 5
-          --shm-size=2g
-      windmill:
-        image: ghcr.io/windmill-labs/windmill-ee:main
-        env:
-          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
-          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
-          WORKER_GROUP: main
-          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
-        options: >-
-          --pull always --health-interval 10s --health-timeout 5s
-          --health-retries 5 --health-cmd "curl
-          http://localhost:8000/api/version"
-        ports:
-          - 8000:8000
-    steps:
-      - uses: denoland/setup-deno@v2
-        with:
-          deno-version: v2.x
-      - name: benchmark
-        timeout-minutes: 30
-        run: deno run  -A -r
-          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          -c
-          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_wac.json
-      - name: Save benchmark results
-        uses: actions/upload-artifact@v4
-        with:
-          name: benchmark_wac
-          path: |
-            *.json
-
   benchmark_graphs:
     runs-on: ubicloud
     needs:
@@ -340,7 +297,6 @@ jobs:
       - benchmark_dedicated
       - benchmark_4workers
       - benchmark_8workers
-      - benchmark_wac
     steps:
       - uses: denoland/setup-deno@v2
         with:

.github/workflows/cli-tests.yml

@@ -1,7 +1,6 @@
 name: CLI Tests
 
 on:
-  workflow_dispatch:
   push:
     branches: [main]
     paths:

.github/workflows/rust-client-check.yml

@@ -18,7 +18,10 @@ jobs:
     runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v31
+      - uses: cachix/install-nix-action@v20
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
       - name: Check rust client builds
         run: cd rust-client && nix develop ../ --command ./dev.nu --check
         timeout-minutes: 16

.github/workflows/rust_on_release.yml

@@ -10,7 +10,10 @@ jobs:
     runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v31
+      - uses: cachix/install-nix-action@v20
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
       - run: cd rust-client && nix develop ../ --command ./dev.nu --check --publish 
         env:
           CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.gitignore

@@ -25,8 +25,6 @@ rust-client/Cargo.toml
 backend/target
 frontend/node_modules
 typescript-client/node_modules
-ai_evals/node_modules
-ai_evals/results/
 frontend/.svelte-kit
 backend/chrome_profiler.json
 .fast-check/

.webmux.yaml

@@ -55,13 +55,11 @@ profiles:
       - id: backend
         kind: command
         split: right
-        workingDir: backend
-        command: PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
+        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/backend" && cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
       - id: frontend
         kind: command
         split: bottom
-        workingDir: frontend
-        command: npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
+        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0
         
   frontendOnly:
     runtime: host
@@ -84,8 +82,7 @@ profiles:
       - id: frontend
         kind: command
         split: right
-        workingDir: frontend
-        command: npm run generate-backend-client && npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
+        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0
 
   agentOnly:
     runtime: host

CHANGELOG.md

@@ -1,175 +1,5 @@
 # Changelog
 
-## [1.668.5](https://github.com/windmill-labs/windmill/compare/v1.668.4...v1.668.5) (2026-03-29)
-
-
-### Bug Fixes
-
-* add per-IP and per-account brute force protection on login endpoint ([#8601](https://github.com/windmill-labs/windmill/issues/8601)) ([06bbe7b](https://github.com/windmill-labs/windmill/commit/06bbe7b94bfb846bd73aaf6abdc83e4c14e70adc))
-* add timestamp validation to webhook signature verification ([#8596](https://github.com/windmill-labs/windmill/issues/8596)) ([74fba2a](https://github.com/windmill-labs/windmill/commit/74fba2abf3dc68b682777c01da360258786fded8))
-* disable workspace webhook events when CLOUD_HOSTED ([#8598](https://github.com/windmill-labs/windmill/issues/8598)) ([be7fbeb](https://github.com/windmill-labs/windmill/commit/be7fbeb8b1f31d15e33b0783b2a504d6a01e532e))
-* harden login rate limiting with CLOUD_HOSTED gating and memory eviction ([#8602](https://github.com/windmill-labs/windmill/issues/8602)) ([754b88a](https://github.com/windmill-labs/windmill/commit/754b88a52c4e76421cb21c1eed87ad9d8385e9aa))
-* prevent SSRF and local file read via git repository resource URLs ([#8600](https://github.com/windmill-labs/windmill/issues/8600)) ([845db72](https://github.com/windmill-labs/windmill/commit/845db72b7344fb87ac9c5e24697750549665c7bf))
-* rename snippet param to avoid svelte compiler shadowing bug in asset usages drawer ([#8595](https://github.com/windmill-labs/windmill/issues/8595)) ([8c770a2](https://github.com/windmill-labs/windmill/commit/8c770a206a3b0704642c0bda2ab2aeb199d8af3f))
-* require mcp: scope for MCP endpoints instead of blanket bypass ([#8597](https://github.com/windmill-labs/windmill/issues/8597)) ([f5fc9f8](https://github.com/windmill-labs/windmill/commit/f5fc9f8485d2ec3e20f8b451305195446b90e5a3))
-* use constant-time comparison for API key and basic auth validation ([#8593](https://github.com/windmill-labs/windmill/issues/8593)) ([b4d1f2a](https://github.com/windmill-labs/windmill/commit/b4d1f2aac789306c2e35e123ac93e12c47c26f99))
-* validate JSON before sql_builder bind to prevent injection via JSONB queries ([#8599](https://github.com/windmill-labs/windmill/issues/8599)) ([970e859](https://github.com/windmill-labs/windmill/commit/970e859a410b0144847a1a30d7059955effdd402))
-
-## [1.668.4](https://github.com/windmill-labs/windmill/compare/v1.668.3...v1.668.4) (2026-03-29)
-
-
-### Bug Fixes
-
-* update git sync version to latest cli ([0549f68](https://github.com/windmill-labs/windmill/commit/0549f682fe14f4d4b2f67941362ed2cc29d974a1))
-
-## [1.668.3](https://github.com/windmill-labs/windmill/compare/v1.668.2...v1.668.3) (2026-03-28)
-
-
-### Bug Fixes
-
-* **cli:** phantom diffs, flow safety, trigger DX, lint watch, error clarity ([#8588](https://github.com/windmill-labs/windmill/issues/8588)) ([c6ce319](https://github.com/windmill-labs/windmill/commit/c6ce3197a72ceeffd702cf2263b1074ecbf1ca33))
-
-## [1.668.2](https://github.com/windmill-labs/windmill/compare/v1.668.1...v1.668.2) (2026-03-28)
-
-
-### Bug Fixes
-
-* **cli:** app push crash, lint path, push --message, run validation, history timestamps ([#8585](https://github.com/windmill-labs/windmill/issues/8585)) ([f40cdaf](https://github.com/windmill-labs/windmill/commit/f40cdaf43453d2643800ed730d6abe6873bbe8e7))
-
-## [1.668.1](https://github.com/windmill-labs/windmill/compare/v1.668.0...v1.668.1) (2026-03-28)
-
-
-### Bug Fixes
-
-* **cli:** fix 13 CLI bugs — exit codes, sync tar fallback, variable encryption, JSON output ([#8582](https://github.com/windmill-labs/windmill/issues/8582)) ([38acaa3](https://github.com/windmill-labs/windmill/commit/38acaa3653728bf9e0ae6f746edf433703b4ab63))
-
-## [1.668.0](https://github.com/windmill-labs/windmill/compare/v1.667.0...v1.668.0) (2026-03-28)
-
-
-### Features
-
-* add DB health diagnostic dashboard for superadmins ([#8574](https://github.com/windmill-labs/windmill/issues/8574)) ([9ceab73](https://github.com/windmill-labs/windmill/commit/9ceab730d7def09c2b46527f8a586789d14f2ce0))
-* **cli:** add job, group, audit, token commands and schedule enable/disable ([#8581](https://github.com/windmill-labs/windmill/issues/8581)) ([d29cb23](https://github.com/windmill-labs/windmill/commit/d29cb234dbff07473b911e5e75e362def8a47650))
-* IAM RDS auth for PostgreSQL worker resources ([#8573](https://github.com/windmill-labs/windmill/issues/8573)) ([56253c0](https://github.com/windmill-labs/windmill/commit/56253c04cb679c58d00750da699a6cb62ed52aca))
-
-
-### Bug Fixes
-
-* add Authority Key Identifier to MITM proxy leaf certs ([#8576](https://github.com/windmill-labs/windmill/issues/8576)) ([ce2e6c8](https://github.com/windmill-labs/windmill/commit/ce2e6c8c015110d0385e6afecdc8313aabca1364))
-* Improve CLI developer experience: error handling, sync workflow, JSON output, workspace forks ([#8578](https://github.com/windmill-labs/windmill/issues/8578)) ([501a4ff](https://github.com/windmill-labs/windmill/commit/501a4ff2a94510145952686d24ccc639781beefe))
-* trigger capture filter and focus issues ([#8579](https://github.com/windmill-labs/windmill/issues/8579)) ([820f28f](https://github.com/windmill-labs/windmill/commit/820f28f8799f8dad5cfab94b51ac9921d664f04a))
-
-## [1.667.0](https://github.com/windmill-labs/windmill/compare/v1.666.0...v1.667.0) (2026-03-27)
-
-
-### Features
-
-* add schedule support to CLI branch-specific items ([#8570](https://github.com/windmill-labs/windmill/issues/8570)) ([b592996](https://github.com/windmill-labs/windmill/commit/b592996eee98ddb664f1b007b95a2096d5d4e3a6))
-* add workspace-level service accounts ([#8560](https://github.com/windmill-labs/windmill/issues/8560)) ([3959fe8](https://github.com/windmill-labs/windmill/commit/3959fe82974f5f0383e94fd83a5d78fe4212d56a))
-* **cli:** generate commented wmill.yaml and add config reference command ([#8546](https://github.com/windmill-labs/windmill/issues/8546)) ([d06b426](https://github.com/windmill-labs/windmill/commit/d06b42613f73c4a7b31c990be22b0c97efab2666))
-* DB-coordinated graceful restart staggering for settings changes ([#8555](https://github.com/windmill-labs/windmill/issues/8555)) ([2f32675](https://github.com/windmill-labs/windmill/commit/2f326758013dd1f1e6ae732e5784a32f1fb6e4bd))
-* improve-replay-ui ([#8250](https://github.com/windmill-labs/windmill/issues/8250)) ([c0aafee](https://github.com/windmill-labs/windmill/commit/c0aafee9a9923d5dc2fa3b99da4378e923933a06))
-* support multiple folder selection in MCP scope selector ([#8557](https://github.com/windmill-labs/windmill/issues/8557)) ([ad19ac9](https://github.com/windmill-labs/windmill/commit/ad19ac9b37b04591c921f93f180bdda961af6cef))
-
-
-### Bug Fixes
-
-* **cli:** preserve inline script files during flow generate-locks ([#8561](https://github.com/windmill-labs/windmill/issues/8561)) ([a8b651d](https://github.com/windmill-labs/windmill/commit/a8b651da9ff86766119e14c0b61652be8a7b453a))
-* emit 0 for OTEL queue metrics when tag queue is empty ([#8559](https://github.com/windmill-labs/windmill/issues/8559)) ([79cc4a9](https://github.com/windmill-labs/windmill/commit/79cc4a92d88486c999799826bd0c9663767103f5))
-* handle inline script deletion in sync push + flow new nonDottedPaths ([#8553](https://github.com/windmill-labs/windmill/issues/8553)) ([943fe9c](https://github.com/windmill-labs/windmill/commit/943fe9c6cc9b046e24007e45b5c37afc4804256a))
-* include importer_kind in dependency debounce key to prevent cross-kind collisions ([#8567](https://github.com/windmill-labs/windmill/issues/8567)) ([bc7007b](https://github.com/windmill-labs/windmill/commit/bc7007bb4265e1f1375c1f0678b74325882a4e92))
-* multi-script dedicated workers race on shared job_dir ([#8551](https://github.com/windmill-labs/windmill/issues/8551)) ([#8569](https://github.com/windmill-labs/windmill/issues/8569)) ([63a3573](https://github.com/windmill-labs/windmill/commit/63a3573951d1f724cc63728ed973d039a5468072))
-* preserve notes on nodes inside collapsed groups ([#8552](https://github.com/windmill-labs/windmill/issues/8552)) ([0fb1153](https://github.com/windmill-labs/windmill/commit/0fb115304afc49812420e9ce24e5048502621059))
-* sanitize flow step summaries for filesystem-safe names ([#8554](https://github.com/windmill-labs/windmill/issues/8554)) ([e15bfbf](https://github.com/windmill-labs/windmill/commit/e15bfbf91ee1517432a6861ebb48e129485006aa))
-* use admin db pool in get_copilot_settings_state ([#8564](https://github.com/windmill-labs/windmill/issues/8564)) ([70f3ee5](https://github.com/windmill-labs/windmill/commit/70f3ee5ed4470e9993be822874f2b38e83a96611))
-
-
-### Performance Improvements
-
-* enable bun bundle caching for WAC v2 scripts ([#8556](https://github.com/windmill-labs/windmill/issues/8556)) ([ab868e9](https://github.com/windmill-labs/windmill/commit/ab868e9ebceadaa55e54770d9d59dc5524da13ff))
-
-## [1.666.0](https://github.com/windmill-labs/windmill/compare/v1.665.0...v1.666.0) (2026-03-26)
-
-
-### Features
-
-* add PDF input support to AI agent ([#8525](https://github.com/windmill-labs/windmill/issues/8525)) ([e44504c](https://github.com/windmill-labs/windmill/commit/e44504c6e93e7a4ee94ced03ab626b79a4fd0754))
-
-
-### Bug Fixes
-
-* add relative imports to the dependency list in deploymentUI ([#8548](https://github.com/windmill-labs/windmill/issues/8548)) ([d760ea5](https://github.com/windmill-labs/windmill/commit/d760ea5eaf4dc33007f1fd3e5e07b86925a0aa11))
-* filter null entries in FileUpload initialValue to prevent s3 access error ([#8544](https://github.com/windmill-labs/windmill/issues/8544)) ([1a73012](https://github.com/windmill-labs/windmill/commit/1a73012e0737a6ebea8307013dc0f79982269d91))
-* pass pre-bound TcpListener to run_server to fix Windows CI test race ([#8542](https://github.com/windmill-labs/windmill/issues/8542)) ([d7f4b95](https://github.com/windmill-labs/windmill/commit/d7f4b950ce6e966ed1b410e03d48fe96bc036e73))
-* resolve parent_hash race condition in sync push with auto_parent ([#8545](https://github.com/windmill-labs/windmill/issues/8545)) ([71549c3](https://github.com/windmill-labs/windmill/commit/71549c3db053bcc209c7065ac8cd42f1e8047cc3))
-* upload_s3_file not working in VS Code extension ([#8547](https://github.com/windmill-labs/windmill/issues/8547)) ([1fa4d91](https://github.com/windmill-labs/windmill/commit/1fa4d919b30ac9eff2d1789fba2695450ba115e7))
-
-## [1.665.0](https://github.com/windmill-labs/windmill/compare/v1.664.0...v1.665.0) (2026-03-26)
-
-
-### Features
-
-* add instance setting to enforce workspace prefix for HTTP routes ([#8528](https://github.com/windmill-labs/windmill/issues/8528)) ([9b3e558](https://github.com/windmill-labs/windmill/commit/9b3e558d84f15052e9c32695a467f8ef7e4ad1f5))
-* add trashbin system for soft-deleting items ([#8519](https://github.com/windmill-labs/windmill/issues/8519)) ([69ce946](https://github.com/windmill-labs/windmill/commit/69ce946241d98ea90bc7135d44ca0c87f928be88))
-* mask sensitive values in job logs ([#8520](https://github.com/windmill-labs/windmill/issues/8520)) ([0885d8c](https://github.com/windmill-labs/windmill/commit/0885d8c986f13ac210e4db3ad38febe9be391ba4))
-* move basic git sync from EE to CE with runtime user count gating ([#8493](https://github.com/windmill-labs/windmill/issues/8493)) ([79d2bd5](https://github.com/windmill-labs/windmill/commit/79d2bd51a00654162754046308d7670242120df6))
-* runner groups for shared-process multi-script dedicated workers ([#8434](https://github.com/windmill-labs/windmill/issues/8434)) ([c28314f](https://github.com/windmill-labs/windmill/commit/c28314f424ea0e04b86565ce88e6c91e0df1a0cf))
-* SCIM user deprovisioning (active:false) + instance-level user disable ([#8484](https://github.com/windmill-labs/windmill/issues/8484)) ([0bd7568](https://github.com/windmill-labs/windmill/commit/0bd756839c0261f255111d62088bdaaecb838085))
-* show groups and notes in flow status viewer ([#8535](https://github.com/windmill-labs/windmill/issues/8535)) ([167084a](https://github.com/windmill-labs/windmill/commit/167084a0ebe73384fa0d31f0b24017a47686a072))
-
-
-### Bug Fixes
-
-* auto-generate datatable SDK reference for app mode system prompt ([#8522](https://github.com/windmill-labs/windmill/issues/8522)) ([8a32322](https://github.com/windmill-labs/windmill/commit/8a32322c187ccc60ec7eafb61a9678f267a82282))
-* consider wmill.yaml environments alias in git sync ([#8532](https://github.com/windmill-labs/windmill/issues/8532)) ([b7475c7](https://github.com/windmill-labs/windmill/commit/b7475c73094a28f520f798f6cb1a0c6b4807ccb7))
-* GitHub Enterprise Server support for self-managed GitHub Apps ([#8507](https://github.com/windmill-labs/windmill/issues/8507)) ([935fb44](https://github.com/windmill-labs/windmill/commit/935fb44c848b8bf9430b5600dd3c3bedb2f89efd))
-* raw apps bundle not found during deployment error ([#8515](https://github.com/windmill-labs/windmill/issues/8515)) ([34e3115](https://github.com/windmill-labs/windmill/commit/34e3115bcbd19a8e0b6f483435586a2ab43d0a8e))
-* require admin for workspace encryption key export ([#8523](https://github.com/windmill-labs/windmill/issues/8523)) ([0317668](https://github.com/windmill-labs/windmill/commit/031766808945aefc926f0836d011c0b2a5d2243d))
-* restrict logout redirect to whitelisted domains ([#8524](https://github.com/windmill-labs/windmill/issues/8524)) ([4c8edd5](https://github.com/windmill-labs/windmill/commit/4c8edd5e944d77ed2d41c2b87171c1115c0fdcdc))
-* serve index disk storage sizes from /srch/ endpoint ([#8511](https://github.com/windmill-labs/windmill/issues/8511)) ([e3620e0](https://github.com/windmill-labs/windmill/commit/e3620e074e1bdb46b2b8d732f35a91d300589663))
-* use /apps_raw/get/ redirect URL for raw apps set as workspace default ([#8508](https://github.com/windmill-labs/windmill/issues/8508)) ([85c52e2](https://github.com/windmill-labs/windmill/commit/85c52e2cded10606cc895d0d3b717e13c69bc9b3))
-* use resource-level scope overrides during OAuth2 token refresh ([#8540](https://github.com/windmill-labs/windmill/issues/8540)) ([55ad0ff](https://github.com/windmill-labs/windmill/commit/55ad0ff5c499c33b766f47c6f32ba5d3eeb14763))
-
-## [1.664.0](https://github.com/windmill-labs/windmill/compare/v1.663.0...v1.664.0) (2026-03-24)
-
-
-### Features
-
-* add instance-level AI settings ([#8453](https://github.com/windmill-labs/windmill/issues/8453)) ([db5e036](https://github.com/windmill-labs/windmill/commit/db5e03610da325288d53afdbca94b9cbfc7ceace))
-* add selfApproval option to WAC + inline approval buttons ([#8440](https://github.com/windmill-labs/windmill/issues/8440)) ([d578e40](https://github.com/windmill-labs/windmill/commit/d578e40101a838d3dffda14157cf72ee4d5a93c0))
-* flow group nodes with collapsible groups ([#8075](https://github.com/windmill-labs/windmill/issues/8075)) ([81eb446](https://github.com/windmill-labs/windmill/commit/81eb446eee359f44374b81320690e5345fd08c15))
-
-
-### Bug Fixes
-
-* add GIT_SSL_CAINFO to tracing proxy env vars ([#8502](https://github.com/windmill-labs/windmill/issues/8502)) ([bdfd5d5](https://github.com/windmill-labs/windmill/commit/bdfd5d57261a4bb760fc57ad41ee56aff9b9c0af))
-* create parent dirs and accept 'python' alias in script bootstrap ([#8497](https://github.com/windmill-labs/windmill/issues/8497)) ([7f27d99](https://github.com/windmill-labs/windmill/commit/7f27d996accb3c3b471d1c50df397867d89c738a))
-
-## [1.663.0](https://github.com/windmill-labs/windmill/compare/v1.662.0...v1.663.0) (2026-03-24)
-
-
-### Features
-
-* add summary field for native triggers ([#8476](https://github.com/windmill-labs/windmill/issues/8476)) ([5089a45](https://github.com/windmill-labs/windmill/commit/5089a458819abbc6f241bc354bebb91520bd1a52))
-* add typed request body to OpenAPI spec generation ([#8481](https://github.com/windmill-labs/windmill/issues/8481)) ([37ebaf4](https://github.com/windmill-labs/windmill/commit/37ebaf4d0ac342703498733f97778a552f979f6a))
-* **cli:** better stale scripts detection [#3](https://github.com/windmill-labs/windmill/issues/3) ([#8480](https://github.com/windmill-labs/windmill/issues/8480)) ([9643006](https://github.com/windmill-labs/windmill/commit/9643006f1e90b991b334bb58caf62301bc26d09d))
-* Debounce node ([#8324](https://github.com/windmill-labs/windmill/issues/8324)) ([5d1c54d](https://github.com/windmill-labs/windmill/commit/5d1c54d9b33d6ff6f2c98481a2740d1e7629cdfa))
-* surface permissioned_as selector in trigger editor UI ([#8475](https://github.com/windmill-labs/windmill/issues/8475)) ([f035b53](https://github.com/windmill-labs/windmill/commit/f035b538bbd786445526339f88be8f33a3628105))
-
-
-### Bug Fixes
-
-* clean up stale dependency map entries for renamed scripts ([#8492](https://github.com/windmill-labs/windmill/issues/8492)) ([47c0c36](https://github.com/windmill-labs/windmill/commit/47c0c363f4fc1d9af7efd07ea172e32989ce50d2))
-* **cli:** add Svelte 5 event delegation guidance and safe push to raw-app skill ([#8466](https://github.com/windmill-labs/windmill/issues/8466)) ([911df95](https://github.com/windmill-labs/windmill/commit/911df958e78d2dab9823dfa7d7e5c9824fc2d565))
-* Fix worker panic when job_isolation changed to unshare at runtime ([#8490](https://github.com/windmill-labs/windmill/issues/8490)) ([cbe47c0](https://github.com/windmill-labs/windmill/commit/cbe47c0b6c22f79452d020777e481ee26970f25b))
-* improve SQS retries ([3c8d351](https://github.com/windmill-labs/windmill/commit/3c8d351c9722a089133871019d27cf3bc3cdc159))
-* Move database manager SQL queries to backend ([#8306](https://github.com/windmill-labs/windmill/issues/8306)) ([aa30fd2](https://github.com/windmill-labs/windmill/commit/aa30fd252dcf40233d191c43a6293fb9feabf010))
-* prevent SQL injection in job query parameters ([#8494](https://github.com/windmill-labs/windmill/issues/8494)) ([54f5a19](https://github.com/windmill-labs/windmill/commit/54f5a19377e9df712e18f85f896e21b1776981ed))
-* respect NO_COLOR env variable for stdout log output ([#8483](https://github.com/windmill-labs/windmill/issues/8483)) ([f329ee7](https://github.com/windmill-labs/windmill/commit/f329ee7aaefbae0ad344743c40825440a936bd30))
-* show effective isolation level on workers page ([#8491](https://github.com/windmill-labs/windmill/issues/8491)) ([37886ed](https://github.com/windmill-labs/windmill/commit/37886edda1443293806a9b1b810196b72e076b12))
-* skip debounce arg accumulation when batch table is empty (CE) ([#8485](https://github.com/windmill-labs/windmill/issues/8485)) ([010753c](https://github.com/windmill-labs/windmill/commit/010753c73ac85237af50acadf9c08567b1bc993c))
-* stop_after_if with empty error_message prevents flow from stopping ([#8464](https://github.com/windmill-labs/windmill/issues/8464)) ([1503bf9](https://github.com/windmill-labs/windmill/commit/1503bf948e3340b8a6933d71885f8f2cb8dc1867))
-
 ## [1.662.0](https://github.com/windmill-labs/windmill/compare/v1.661.0...v1.662.0) (2026-03-20)
 
 

ai_evals/.gitignore

@@ -1,2 +0,0 @@
-.env
-results/

ai_evals/README.md

@@ -1,168 +0,0 @@
-# AI Evals
-
-Small benchmark runner for the four Windmill AI generation modes:
-
-- `cli`
-- `flow`
-- `script`
-- `app`
-
-The benchmark always tests the current production prompts, tools, and guidance in this checkout.
-
-Each attempt runs:
-
-1. the real production path
-2. deterministic validation
-3. LLM judging
-
-## Install
-
-```bash
-cd ai_evals
-bun install
-```
-
-Frontend modes also require frontend dependencies:
-
-```bash
-cd frontend
-bun install
-```
-
-## Commands
-
-List model aliases:
-
-```bash
-cd ai_evals
-bun run cli -- models
-```
-
-List cases:
-
-```bash
-cd ai_evals
-bun run cli -- cases
-bun run cli -- cases flow
-```
-
-Run benchmarks:
-
-```bash
-cd ai_evals
-bun run cli -- run flow
-bun run cli -- run flow flow-test4-order-processing-loop --model opus
-bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
-bun run cli -- run flow --record
-bun run cli -- run cli bun-hello-script
-```
-
-Public CLI surface:
-
-- `models`
-- `cases [mode]`
-- `run <mode> [caseIds...]`
-
-`run` options:
-
-- `--runs <n>`: repeat each case `n` times
-- `--output <path>`: custom result JSON path
-- `--model <alias>`: choose the model under test
-- `--verbose`: stream assistant output for frontend runs
-- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl`
-
-## Models
-
-Use `bun run cli -- models` to see the current aliases.
-
-Today:
-
-- `haiku`
-- `sonnet`
-- `opus`
-- `4o`
-
-Notes:
-
-- the command also prints accepted alias spellings such as `gpt-4o`, `claude-opus-4.6`, and `claude-haiku-4.5`
-- frontend modes (`flow`, `script`, `app`) can use Anthropic and OpenAI-backed aliases
-- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
-- the judge model is separate and currently defaults to `claude-sonnet-4-6`
-
-## Case Format
-
-Cases live in one YAML file per mode under `ai_evals/cases/`.
-
-Minimal shape:
-
-```yaml
-- id: flow-test0-sum-two-numbers
-  prompt: |-
-    Create a flow that takes two numbers, `a` and `b`, and returns their sum.
-  initial: ai_evals/fixtures/...
-  expected: ai_evals/fixtures/...
-```
-
-Optional fields:
-
-- `initial`: starting state fixture
-- `expected`: expected artifact fixture
-- `validate`: extra deterministic validation rules
-
-For `flow` mode, `validate` can express requirements such as:
-
-- accepted input schema shapes
-- required `results.*` reference validity
-- required module/code/input characteristics
-
-For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
-existing scripts and flows. That lets the real `search_workspace` and
-`get_runnable_details` tools discover reusable workspace runnables during evals.
-
-## Results And Artifacts
-
-Every run writes:
-
-- a summary JSON under `ai_evals/results/`
-- generated artifacts in a sibling directory
-
-If `--record` is used, the CLI also appends one compact JSON line to:
-
-- `ai_evals/history/flow.jsonl`
-- `ai_evals/history/script.jsonl`
-- `ai_evals/history/app.jsonl`
-- `ai_evals/history/cli.jsonl`
-
-Each recorded line contains:
-
-- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
-- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`)
-- `failedCaseIds`
-
-Example:
-
-- summary: `ai_evals/results/2026-04-09T09-40-33.051Z__flow.json`
-- artifacts: `ai_evals/results/2026-04-09T09-40-33.051Z__flow/`
-
-Typical artifacts by mode:
-
-- `flow`: `flow.json`
-- `script`: `script.json` plus the generated script file
-- `app`: `app.json` plus frontend/backend files
-- `cli`: `assistant-output.txt` plus generated workspace files
-
-## Layout
-
-- `cases/`: one YAML file per mode
-- `fixtures/`: initial and expected fixtures
-- `core/`: shared loading, model resolution, validation, judging, and result writing
-- `modes/`: one runner per mode
-- `history/`: optional tracked pass-rate history written by `run --record`, one JSONL file per mode
-- `results/`: local benchmark output and artifacts
-
-## Notes
-
-- Frontend modes reuse the production frontend chat code through the Vitest bridge.
-- CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / `AGENTS.md` flow.
-- Frontend progress streams live while the benchmark is running.
-- Deterministic validators should stay focused on real correctness constraints, not one exact implementation shape.

ai_evals/adapters/cli/runtime.ts

@@ -1,107 +0,0 @@
-import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models";
-
-export interface ToolInvocation {
-  tool: string;
-  input: Record<string, unknown>;
-  timestamp: number;
-}
-
-export interface PromptRunResult {
-  toolsUsed: ToolInvocation[];
-  skillsInvoked: string[];
-  output: string;
-  durationMs: number;
-  assistantMessageCount: number;
-}
-
-const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
-export const DEFAULT_CLI_EVAL_MODEL: CliEvalModelConfig = getCliEvalModel(resolveEvalModel("cli"));
-
-export function getGeneratedSkillsSource(): string {
-  return join(REPO_ROOT, "system_prompts", "auto-generated", "skills");
-}
-
-export async function runPromptAndCapture(
-  prompt: string,
-  cwd: string,
-  maxTurns: number = 3,
-  modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
-): Promise<PromptRunResult> {
-  const toolsUsed: ToolInvocation[] = [];
-  const skillsInvoked: string[] = [];
-  let output = "";
-  let assistantMessageCount = 0;
-  const startedAt = Date.now();
-
-  const options: Options = {
-    cwd,
-    model: modelConfig.model,
-    maxTurns,
-    settingSources: ["project"],
-    allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"]
-  };
-
-  for await (const message of query({ prompt, options })) {
-    if (message.type === "assistant") {
-      assistantMessageCount += 1;
-      const content = message.message?.content;
-      if (Array.isArray(content)) {
-        for (const block of content) {
-          if (block.type === "tool_use") {
-            toolsUsed.push({
-              tool: block.name,
-              input: block.input as Record<string, unknown>,
-              timestamp: Date.now()
-            });
-
-            if (block.name === "Skill" && typeof block.input === "object" && block.input !== null) {
-              const skillInput = block.input as { skill?: string };
-              if (skillInput.skill) {
-                skillsInvoked.push(skillInput.skill);
-              }
-            }
-          } else if (block.type === "text") {
-            output += block.text;
-          }
-        }
-      }
-    } else if (message.type === "result") {
-      const resultMessage = message as { result?: string };
-      if (typeof resultMessage.result === "string") {
-        output += resultMessage.result;
-      }
-    }
-  }
-
-  return {
-    toolsUsed,
-    skillsInvoked,
-    output,
-    durationMs: Date.now() - startedAt,
-    assistantMessageCount
-  };
-}
-
-export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean {
-  return result.skillsInvoked.some((skill) => skill === skillName || skill.includes(skillName));
-}
-
-export function wasToolUsed(result: PromptRunResult, toolName: string): boolean {
-  return result.toolsUsed.some((tool) => tool.tool === toolName);
-}
-
-export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string {
-  return `${modelConfig.provider}:${modelConfig.model}`;
-}
-
-export function getToolInputs(
-  result: PromptRunResult,
-  toolName: string
-): Record<string, unknown>[] {
-  return result.toolsUsed
-    .filter((tool) => tool.tool === toolName)
-    .map((tool) => tool.input);
-}

ai_evals/adapters/frontend/benchmarkRunner.ts

@@ -1,87 +0,0 @@
-import { loadSelectedCases } from "../../core/cases";
-import {
-  formatRunModelLabel,
-  getFrontendEvalModel,
-  resolveEvalModel,
-} from "../../core/models";
-import { buildRunResult } from "../../core/results";
-import { runSuite } from "../../core/runSuite";
-import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
-import { emitFrontendBenchmarkProgress } from "./progress";
-import { createAppModeRunner } from "../../modes/app";
-import { createFlowModeRunner } from "../../modes/flow";
-import { createScriptModeRunner } from "../../modes/script";
-import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
-
-export type FrontendBenchmarkMode = "flow" | "app" | "script";
-
-export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
-  const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
-  const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS);
-  const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS");
-  const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
-  const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1";
-  const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL);
-
-  const selectedCases = await loadSelectedCases(mode, caseIds);
-  const modeRunner = getModeRunner(mode, getFrontendEvalModel(model));
-  const runModel = formatRunModelLabel(mode, model);
-  const caseResults = await runSuite({
-    modeRunner,
-    cases: selectedCases,
-    runs,
-    runModel,
-    judgeModel: DEFAULT_JUDGE_MODEL,
-    concurrency: verbose ? 1 : undefined,
-    verbose,
-    onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined,
-  });
-
-  return buildRunResult({
-    mode,
-    runs,
-    runModel,
-    judgeModel: DEFAULT_JUDGE_MODEL,
-    caseResults,
-  });
-}
-
-function getModeRunner(
-  mode: FrontendBenchmarkMode,
-  model: ReturnType<typeof getFrontendEvalModel>
-): ModeRunner<any, any, any> {
-  switch (mode) {
-    case "flow":
-      return createFlowModeRunner(model);
-    case "app":
-      return createAppModeRunner(model);
-    case "script":
-      return createScriptModeRunner(model);
-  }
-}
-
-function parseMode(value: string | undefined): FrontendBenchmarkMode {
-  if (value === "flow" || value === "app" || value === "script") {
-    return value;
-  }
-  throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);
-}
-
-function parseOptionalJsonStringArray(value: string | undefined): string[] {
-  if (!value) {
-    return [];
-  }
-  const parsed = JSON.parse(value) as unknown;
-  if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== "string")) {
-    throw new Error("WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array");
-  }
-  return parsed;
-}
-
-function parsePositiveInteger(value: string | undefined, envName: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed <= 0) {
-    throw new Error(`${envName} must be a positive integer`);
-  }
-  return parsed;
-}

ai_evals/adapters/frontend/core/app/appEvalRunner.ts

@@ -1,89 +0,0 @@
-import { mkdtemp } from 'fs/promises'
-import { tmpdir } from 'os'
-import { join } from 'path'
-import type {
-	AppFiles,
-	BackendRunnable,
-	AppAIChatHelpers
-} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
-import {
-	getAppTools,
-	prepareAppSystemMessage,
-	prepareAppUserMessage
-} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
-import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import { createAppFileHelpers } from './fileHelpers'
-import { runEval } from '../shared'
-import type { AIProvider } from '$lib/gen/types.gen'
-import type { ModeRunContext } from '../../../../core/types'
-
-export interface AppEvalResult {
-	success: boolean
-	files: AppFiles
-	error?: string
-	assistantMessageCount: number
-	toolCallCount: number
-	toolsUsed: string[]
-}
-
-export interface AppEvalOptions {
-	initialFrontend?: Record<string, string>
-	initialBackend?: Record<string, BackendRunnable>
-	model?: string
-	maxIterations?: number
-	provider?: AIProvider
-	workspaceRoot?: string
-	runContext?: ModeRunContext
-}
-
-export async function runAppEval(
-	userPrompt: string,
-	apiKey: string,
-	options?: AppEvalOptions
-): Promise<AppEvalResult> {
-	const workspaceRoot =
-		options?.workspaceRoot ??
-		(await mkdtemp(join(tmpdir(), 'wmill-frontend-app-benchmark-')))
-	const { helpers, getFiles, cleanup } = await createAppFileHelpers(
-		options?.initialFrontend ?? {},
-		options?.initialBackend ?? {},
-		workspaceRoot
-	)
-
-	try {
-		const systemMessage = prepareAppSystemMessage()
-		const tools = getAppTools() as ProductionTool<AppAIChatHelpers>[]
-		const model = options?.model ?? 'claude-haiku-4-5-20251001'
-		const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext())
-
-		const rawResult = await runEval({
-			userPrompt,
-			systemMessage,
-			userMessage,
-			tools,
-			helpers,
-			apiKey,
-			getOutput: getFiles,
-			onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
-			onAssistantToken: options?.runContext?.onAssistantChunk,
-			onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
-			options: {
-				maxIterations: options?.maxIterations,
-				model,
-				workspace: workspaceRoot,
-				provider: options?.provider
-			}
-		})
-
-		return {
-			files: rawResult.output,
-			success: rawResult.success,
-			error: rawResult.error,
-			assistantMessageCount: rawResult.iterations,
-			toolCallCount: rawResult.toolCallsCount,
-			toolsUsed: rawResult.toolsCalled
-		}
-	} finally {
-		await cleanup()
-	}
-}

ai_evals/adapters/frontend/core/app/fileHelpers.ts

@@ -1,255 +0,0 @@
-import { mkdir, rm, writeFile } from 'fs/promises'
-import { dirname, join } from 'path'
-import type {
-	AppAIChatHelpers,
-	AppFiles,
-	BackendRunnable,
-	DataTableSchema,
-	LintResult,
-	SelectedContext
-} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
-
-function createEmptyLintResult(): LintResult {
-	return {
-		errorCount: 0,
-		warningCount: 0,
-		errors: { frontend: {}, backend: {} },
-		warnings: { frontend: {}, backend: {} }
-	}
-}
-
-async function writeFrontendFile(
-	workspaceRoot: string | undefined,
-	path: string,
-	content: string
-): Promise<void> {
-	if (!workspaceRoot) {
-		return
-	}
-	const relativePath = path.startsWith('/') ? path.slice(1) : path
-	const fullPath = join(workspaceRoot, 'frontend', relativePath)
-	await mkdir(dirname(fullPath), { recursive: true })
-	await writeFile(fullPath, content, 'utf8')
-}
-
-async function removeFrontendFile(workspaceRoot: string | undefined, path: string): Promise<void> {
-	if (!workspaceRoot) {
-		return
-	}
-	const relativePath = path.startsWith('/') ? path.slice(1) : path
-	await rm(join(workspaceRoot, 'frontend', relativePath), { force: true })
-}
-
-async function writeBackendRunnable(
-	workspaceRoot: string | undefined,
-	key: string,
-	runnable: BackendRunnable
-): Promise<void> {
-	if (!workspaceRoot) {
-		return
-	}
-	const runnableDir = join(workspaceRoot, 'backend', key)
-	await mkdir(runnableDir, { recursive: true })
-
-	const meta: { name: string; language?: string; type?: string; path?: string } = {
-		name: runnable.name
-	}
-
-	if (runnable.type === 'inline' && runnable.inlineScript) {
-		meta.language = runnable.inlineScript.language
-		const extension = runnable.inlineScript.language === 'python3' ? 'py' : 'ts'
-		await writeFile(
-			join(runnableDir, `main.${extension}`),
-			runnable.inlineScript.content,
-			'utf8'
-		)
-	} else {
-		meta.type = runnable.type
-		if (runnable.path) {
-			meta.path = runnable.path
-		}
-	}
-
-	await writeFile(join(runnableDir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n', 'utf8')
-}
-
-async function removeBackendRunnable(workspaceRoot: string | undefined, key: string): Promise<void> {
-	if (!workspaceRoot) {
-		return
-	}
-	await rm(join(workspaceRoot, 'backend', key), { recursive: true, force: true })
-}
-
-async function persistDatatables(
-	workspaceRoot: string | undefined,
-	datatables: DataTableSchema[]
-): Promise<void> {
-	if (!workspaceRoot) {
-		return
-	}
-	await writeFile(
-		join(workspaceRoot, 'datatables.json'),
-		JSON.stringify(datatables, null, 2) + '\n',
-		'utf8'
-	)
-}
-
-export async function createAppFileHelpers(
-	initialFrontend: Record<string, string> = {},
-	initialBackend: Record<string, BackendRunnable> = {},
-	workspaceRoot?: string
-): Promise<{
-	helpers: AppAIChatHelpers
-	getFiles: () => AppFiles
-	getFrontend: () => Record<string, string>
-	getBackend: () => Record<string, BackendRunnable>
-	cleanup: () => Promise<void>
-	workspaceDir: string | null
-}> {
-	let frontend = { ...initialFrontend }
-	let backend = { ...initialBackend }
-	let snapshotId = 0
-	const snapshots = new Map<
-		number,
-		{ frontend: Record<string, string>; backend: Record<string, BackendRunnable> }
-	>()
-	const datatables: DataTableSchema[] = []
-
-	for (const [path, content] of Object.entries(frontend)) {
-		await writeFrontendFile(workspaceRoot, path, content)
-	}
-	for (const [key, runnable] of Object.entries(backend)) {
-		await writeBackendRunnable(workspaceRoot, key, runnable)
-	}
-	await persistDatatables(workspaceRoot, datatables)
-
-	const helpers: AppAIChatHelpers = {
-		listFrontendFiles: () => Object.keys(frontend),
-		getFrontendFile: (path: string) => frontend[path],
-		getFrontendFiles: () => ({ ...frontend }),
-		setFrontendFile: (path: string, content: string) => {
-			frontend[path] = content
-			void writeFrontendFile(workspaceRoot, path, content)
-			return createEmptyLintResult()
-		},
-		deleteFrontendFile: (path: string) => {
-			delete frontend[path]
-			void removeFrontendFile(workspaceRoot, path)
-		},
-		listBackendRunnables: () =>
-			Object.entries(backend).map(([key, runnable]) => ({
-				key,
-				name: runnable.name
-			})),
-		getBackendRunnable: (key: string) => backend[key],
-		getBackendRunnables: () => ({ ...backend }),
-		setBackendRunnable: async (key: string, runnable: BackendRunnable) => {
-			backend[key] = runnable
-			await writeBackendRunnable(workspaceRoot, key, runnable)
-			return createEmptyLintResult()
-		},
-		deleteBackendRunnable: (key: string) => {
-			delete backend[key]
-			void removeBackendRunnable(workspaceRoot, key)
-		},
-		getFiles: (): AppFiles => ({
-			frontend: { ...frontend },
-			backend: { ...backend }
-		}),
-		getSelectedContext: (): SelectedContext => ({ type: 'none' }),
-		snapshot: () => {
-			const id = ++snapshotId
-			snapshots.set(id, {
-				frontend: { ...frontend },
-				backend: { ...backend }
-			})
-			return id
-		},
-		revertToSnapshot: (id: number) => {
-			const snapshot = snapshots.get(id)
-			if (!snapshot) {
-				return
-			}
-			frontend = { ...snapshot.frontend }
-			backend = { ...snapshot.backend }
-			void syncWorkspace()
-		},
-		lint: () => createEmptyLintResult(),
-		getDatatables: async () => structuredClone(datatables),
-		getAvailableDatatableNames: () => datatables.map((datatable) => datatable.datatable_name),
-		execDatatableSql: async (
-			datatableName: string,
-			sql: string,
-			newTable?: { schema: string; name: string }
-		) => {
-			if (newTable) {
-				datatables.push({
-					datatable_name: datatableName,
-					schemas: {
-						[newTable.schema]: {
-							[newTable.name]: {}
-						}
-					}
-				})
-				await persistDatatables(workspaceRoot, datatables)
-			}
-			return {
-				success: true,
-				result: [
-					{
-						datatableName,
-						sql
-					}
-				]
-			}
-		},
-		addTableToWhitelist: (datatableName: string, schemaName: string, tableName: string) => {
-			const existing = datatables.find((entry) => entry.datatable_name === datatableName)
-			if (existing) {
-				existing.schemas[schemaName] ??= {}
-				existing.schemas[schemaName][tableName] ??= {}
-			} else {
-				datatables.push({
-					datatable_name: datatableName,
-					schemas: {
-						[schemaName]: {
-							[tableName]: {}
-						}
-					}
-				})
-			}
-			void persistDatatables(workspaceRoot, datatables)
-		}
-	}
-
-	async function syncWorkspace(): Promise<void> {
-		if (!workspaceRoot) {
-			return
-		}
-		await rm(join(workspaceRoot, 'frontend'), { recursive: true, force: true })
-		await rm(join(workspaceRoot, 'backend'), { recursive: true, force: true })
-		for (const [path, content] of Object.entries(frontend)) {
-			await writeFrontendFile(workspaceRoot, path, content)
-		}
-		for (const [key, runnable] of Object.entries(backend)) {
-			await writeBackendRunnable(workspaceRoot, key, runnable)
-		}
-		await persistDatatables(workspaceRoot, datatables)
-	}
-
-	return {
-		helpers,
-		getFiles: () => ({
-			frontend: { ...frontend },
-			backend: { ...backend }
-		}),
-		getFrontend: () => ({ ...frontend }),
-		getBackend: () => ({ ...backend }),
-		cleanup: async () => {
-			if (workspaceRoot) {
-				await rm(workspaceRoot, { recursive: true, force: true })
-			}
-		},
-		workspaceDir: workspaceRoot ?? null
-	}
-}

ai_evals/adapters/frontend/core/flow/fileHelpers.ts

@@ -1,147 +0,0 @@
-import { mkdir, rm, writeFile } from 'fs/promises'
-import { dirname, join } from 'path'
-import type { FlowModule, InputTransform } from '../../../../../frontend/src/lib/gen'
-import type { ExtendedOpenFlow } from '../../../../../frontend/src/lib/components/flows/types'
-import type { FlowAIChatHelpers } from '../../../../../frontend/src/lib/components/copilot/chat/flow/core'
-import type { ScriptLintResult } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import { findModuleById } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import {
-	createInlineScriptSession
-} from '../../../../../frontend/src/lib/components/copilot/chat/flow/inlineScriptsUtils'
-import {
-	registerBenchmarkWorkspaceRunnables,
-	unregisterBenchmarkWorkspaceRunnables,
-	type BenchmarkWorkspaceFlow,
-	type BenchmarkWorkspaceScript
-} from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-
-const EMPTY_SCRIPT_LINT_RESULT: ScriptLintResult = {
-	errorCount: 0,
-	warningCount: 0,
-	errors: [],
-	warnings: []
-}
-
-export interface FlowWorkspaceFixtures {
-	scripts?: BenchmarkWorkspaceScript[]
-	flows?: BenchmarkWorkspaceFlow[]
-}
-
-export async function createFlowFileHelpers(
-	initialModules: FlowModule[] = [],
-	initialSchema?: Record<string, any>,
-	workspaceRoot?: string,
-	workspaceFixtures?: FlowWorkspaceFixtures
-): Promise<{
-	helpers: FlowAIChatHelpers
-	getFlow: () => ExtendedOpenFlow
-	getModules: () => FlowModule[]
-	cleanup: () => Promise<void>
-	workspaceDir: string | null
-}> {
-	let flow: ExtendedOpenFlow = {
-		value: { modules: structuredClone(initialModules) },
-		summary: '',
-		schema: initialSchema ?? {
-			$schema: 'https://json-schema.org/draft/2020-12/schema',
-			properties: {},
-			required: [],
-			type: 'object'
-		}
-	}
-	const inlineScriptSession = createInlineScriptSession()
-
-	const flowFilePath = workspaceRoot ? join(workspaceRoot, 'flow.json') : null
-
-	async function persistFlow(): Promise<void> {
-		if (!flowFilePath) {
-			return
-		}
-		await mkdir(dirname(flowFilePath), { recursive: true })
-		await writeFile(flowFilePath, JSON.stringify(flow, null, 2) + '\n', 'utf8')
-	}
-
-	await persistFlow()
-
-	if (workspaceRoot && workspaceFixtures) {
-		registerBenchmarkWorkspaceRunnables(workspaceRoot, workspaceFixtures)
-	}
-
-	const helpers: FlowAIChatHelpers = {
-		getFlowAndSelectedId: () => ({ flow, selectedId: '' }),
-		getModules: (id?: string) => {
-			if (!id) return flow.value.modules
-			const module = findModuleById(flow.value.modules, id)
-			return module ? [module] : []
-		},
-		inlineScriptSession,
-		setSnapshot: () => {},
-		revertToSnapshot: () => {},
-		setCode: async (id: string, code: string) => {
-			const module = findModuleById(flow.value.modules, id)
-			if (module && module.value.type === 'rawscript') {
-				module.value.content = code
-			}
-			inlineScriptSession.set(id, code)
-			await persistFlow()
-		},
-		setFlowJson: async (
-			modules: FlowModule[] | undefined,
-			schema: Record<string, any> | undefined
-		) => {
-			if (modules) {
-				flow.value.modules = inlineScriptSession.restoreInlineScriptReferences(modules)
-				const unresolvedRefs = inlineScriptSession.findUnresolvedInlineScriptRefs(flow.value.modules)
-				if (unresolvedRefs.length > 0) {
-					throw new Error(
-						`Unresolved inline script references: ${unresolvedRefs.join(', ')}`
-					)
-				}
-			}
-			if (schema !== undefined) {
-				flow.schema = schema
-			}
-			await persistFlow()
-		},
-		getFlowInputsSchema: async () => flow.schema ?? {},
-		updateExprsToSet: (_id: string, _inputTransforms: Record<string, InputTransform>) => {},
-		acceptAllModuleActions: () => {},
-		rejectAllModuleActions: () => {},
-		hasPendingChanges: () => false,
-		selectStep: (_id: string) => {},
-		testFlow: async (args?: Record<string, any>) => {
-			if (workspaceRoot) {
-				const runPath = join(workspaceRoot, 'test-run.json')
-				await writeFile(
-					runPath,
-					JSON.stringify(
-						{
-							requestedArgs: args ?? {},
-							modules: flow.value.modules.map((module) => module.id)
-						},
-						null,
-						2
-					) + '\n',
-					'utf8'
-				)
-			}
-			return `mock-job-id-${Date.now()}`
-		},
-		getLintErrors: async () => EMPTY_SCRIPT_LINT_RESULT
-	}
-
-	return {
-		helpers,
-		getFlow: () => flow,
-		getModules: () => flow.value.modules,
-		cleanup: async () => {
-			if (workspaceRoot) {
-				unregisterBenchmarkWorkspaceRunnables(workspaceRoot)
-			}
-			if (workspaceRoot) {
-				await rm(workspaceRoot, { recursive: true, force: true })
-			}
-		},
-		workspaceDir: workspaceRoot ?? null
-	}
-}

ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts

@@ -1,100 +0,0 @@
-import { mkdtemp } from 'fs/promises'
-import { tmpdir } from 'os'
-import { join } from 'path'
-import type { FlowModule } from '$lib/gen'
-import type { AIProvider } from '$lib/gen/types.gen'
-import type { ExtendedOpenFlow } from '$lib/components/flows/types'
-import {
-	flowTools,
-	prepareFlowSystemMessage,
-	prepareFlowUserMessage,
-	type FlowAIChatHelpers
-} from '../../../../../frontend/src/lib/components/copilot/chat/flow/core'
-import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import { createFlowFileHelpers, type FlowWorkspaceFixtures } from './fileHelpers'
-import { runEval } from '../shared'
-import type { ModeRunContext } from '../../../../core/types'
-
-export interface FlowFixture {
-	value?: {
-		modules?: FlowModule[]
-	}
-	schema?: Record<string, unknown>
-}
-
-export interface FlowEvalResult {
-	success: boolean
-	flow: ExtendedOpenFlow
-	error?: string
-	assistantMessageCount: number
-	toolCallCount: number
-	toolsUsed: string[]
-}
-
-export interface FlowEvalOptions {
-	initialFlow?: FlowFixture
-	workspaceFixtures?: FlowWorkspaceFixtures
-	model?: string
-	maxIterations?: number
-	provider?: AIProvider
-	workspaceRoot?: string
-	runContext?: ModeRunContext
-}
-
-export async function runFlowEval(
-	userPrompt: string,
-	apiKey: string,
-	options?: FlowEvalOptions
-): Promise<FlowEvalResult> {
-	const workspaceRoot =
-		options?.workspaceRoot ??
-		(await mkdtemp(join(tmpdir(), 'wmill-frontend-flow-benchmark-')))
-	const { helpers, getFlow, cleanup } = await createFlowFileHelpers(
-		options?.initialFlow?.value?.modules ?? [],
-		options?.initialFlow?.schema,
-		workspaceRoot,
-		options?.workspaceFixtures
-	)
-
-	try {
-		const systemMessage = prepareFlowSystemMessage()
-		const tools = flowTools as ProductionTool<FlowAIChatHelpers>[]
-		const model = options?.model ?? 'claude-haiku-4-5-20251001'
-		const userMessage = prepareFlowUserMessage(
-			userPrompt,
-			helpers.getFlowAndSelectedId(),
-			[],
-			helpers.inlineScriptSession
-		)
-
-		const rawResult = await runEval({
-			userPrompt,
-			systemMessage,
-			userMessage,
-			tools,
-			helpers,
-			apiKey,
-			getOutput: getFlow,
-			onAssistantMessageStart: options?.runContext?.onAssistantMessageStart,
-			onAssistantToken: options?.runContext?.onAssistantChunk,
-			onAssistantMessageEnd: options?.runContext?.onAssistantMessageEnd,
-			options: {
-				maxIterations: options?.maxIterations,
-				model,
-				workspace: workspaceRoot,
-				provider: options?.provider
-			}
-		})
-
-		return {
-			flow: rawResult.output,
-			success: rawResult.success,
-			error: rawResult.error,
-			assistantMessageCount: rawResult.iterations,
-			toolCallCount: rawResult.toolCallsCount,
-			toolsUsed: rawResult.toolsCalled
-		}
-	} finally {
-		await cleanup()
-	}
-}

ai_evals/adapters/frontend/core/script/fileHelpers.ts

@@ -1,200 +0,0 @@
-import { mkdir, rm, writeFile } from 'fs/promises'
-import { dirname, join } from 'path'
-import ts from 'typescript'
-import type { ScriptLang } from '../../../../../frontend/src/lib/gen/types.gen'
-import type { ReviewChangesOpts } from '../../../../../frontend/src/lib/components/copilot/chat/monaco-adapter'
-import type { ScriptLintResult } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import type { ScriptChatHelpers } from '../../../../../frontend/src/lib/components/copilot/chat/script/core'
-
-export interface ScriptEvalState {
-	code: string
-	lang: ScriptLang | 'bunnative'
-	path: string
-	args: Record<string, any>
-}
-
-const TS_LIKE_LANGUAGES = new Set<ScriptLang | 'bunnative'>(['bun', 'deno', 'nativets', 'bunnative'])
-const JS_LIKE_LANGUAGES = new Set<ScriptLang | 'bunnative'>(['bun', 'deno', 'nativets', 'bunnative'])
-
-function hasSupportedEntrypoint(code: string): boolean {
-	return (
-		/export\s+(async\s+)?function\s+main\s*\(/.test(code) ||
-		/export\s+(async\s+)?function\s+preprocessor\s*\(/.test(code)
-	)
-}
-
-function compilerOptionsForLanguage(lang: ScriptLang | 'bunnative'): ts.CompilerOptions | null {
-	if (!TS_LIKE_LANGUAGES.has(lang)) {
-		return null
-	}
-
-	return {
-		target: ts.ScriptTarget.ES2022,
-		module: ts.ModuleKind.ESNext,
-		moduleResolution: ts.ModuleResolutionKind.Bundler,
-		noEmit: true,
-		allowJs: true,
-		checkJs: false,
-		strict: false,
-		skipLibCheck: true
-	}
-}
-
-function getLineAndColumn(sourceText: string, start: number): { line: number; column: number } {
-	const prefix = sourceText.slice(0, Math.max(0, start))
-	const line = prefix.split('\n').length
-	const lastNewline = prefix.lastIndexOf('\n')
-	const column = lastNewline === -1 ? prefix.length + 1 : prefix.length - lastNewline
-	return { line, column }
-}
-
-function buildLintResult(code: string, lang: ScriptLang | 'bunnative'): ScriptLintResult {
-	const diagnostics: ScriptLintResult['errors'] = []
-	const compilerOptions = compilerOptionsForLanguage(lang)
-
-	if (compilerOptions) {
-		const sourceFile = ts.createSourceFile(
-			lang === 'deno' ? 'script.ts' : 'script.ts',
-			code,
-			ts.ScriptTarget.ES2022,
-			true,
-			JS_LIKE_LANGUAGES.has(lang) ? ts.ScriptKind.TS : ts.ScriptKind.JS
-		)
-		const output = ts.transpileModule(code, {
-			compilerOptions,
-			fileName: sourceFile.fileName,
-			reportDiagnostics: true
-		})
-
-		for (const diagnostic of output.diagnostics ?? []) {
-			const start = diagnostic.start ?? 0
-			const length = diagnostic.length ?? 1
-			const { line, column } = getLineAndColumn(code, start)
-			const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')
-			diagnostics.push({
-				startLineNumber: line,
-				startColumn: column,
-				endLineNumber: line,
-				endColumn: column + Math.max(1, length),
-				message,
-				severity: 8
-			} as ScriptLintResult['errors'][number])
-		}
-	}
-
-	if (!hasSupportedEntrypoint(code)) {
-		diagnostics.push({
-			startLineNumber: 1,
-			startColumn: 1,
-			endLineNumber: 1,
-			endColumn: 1,
-			message: 'Script must export a main or preprocessor function.',
-			severity: 8
-		} as ScriptLintResult['errors'][number])
-	}
-
-	return {
-		errorCount: diagnostics.length,
-		warningCount: 0,
-		errors: diagnostics,
-		warnings: []
-	}
-}
-
-function formatScriptTestSummary(success: boolean, result: Record<string, unknown>): string {
-	return `Result (${success ? 'SUCCESS' : 'FAILED'})\n\n${JSON.stringify(result, null, 2)}\n\nLogs:\n\nNo logs available`
-}
-
-export async function createScriptFileHelpers(
-	initialScript: ScriptEvalState,
-	workspaceRoot?: string
-): Promise<{
-	helpers: ScriptChatHelpers
-	getScript: () => ScriptEvalState
-	cleanup: () => Promise<void>
-	workspaceDir: string | null
-}> {
-	let script = structuredClone(initialScript)
-	const scriptFilePath = workspaceRoot ? join(workspaceRoot, script.path) : null
-
-	async function persistScript(): Promise<void> {
-		if (!scriptFilePath) {
-			return
-		}
-		await mkdir(dirname(scriptFilePath), { recursive: true })
-		await writeFile(scriptFilePath, script.code, 'utf8')
-	}
-
-	await persistScript()
-
-	const helpers: ScriptChatHelpers = {
-		getScriptOptions: () => ({
-			code: script.code,
-			lang: script.lang,
-			path: script.path,
-			args: structuredClone(script.args)
-		}),
-		applyCode: async (code: string, opts?: ReviewChangesOpts) => {
-			if (opts?.mode === 'revert') {
-				return
-			}
-			script = {
-				...script,
-				code
-			}
-			await persistScript()
-		},
-		getLintErrors: () => buildLintResult(script.code, script.lang),
-		runScriptTest: async (args: Record<string, any>) => {
-			const lintResult = buildLintResult(script.code, script.lang)
-			const success = lintResult.errorCount === 0
-
-			if (workspaceRoot) {
-				await writeFile(
-					join(workspaceRoot, 'test-run.json'),
-					JSON.stringify(
-						{
-							requestedArgs: args,
-							success,
-							errorCount: lintResult.errorCount,
-							path: script.path
-						},
-						null,
-						2
-					) + '\n',
-					'utf8'
-				)
-			}
-
-			return formatScriptTestSummary(
-				success,
-				success
-					? {
-							path: script.path,
-							args,
-							validated: true
-						}
-					: {
-							path: script.path,
-							args,
-							errorCount: lintResult.errorCount,
-							errors: lintResult.errors.map((entry) => ({
-								line: entry.startLineNumber,
-								message: entry.message
-							}))
-						}
-			)
-		}
-	}
-
-	return {
-		helpers,
-		getScript: () => structuredClone(script),
-		cleanup: async () => {
-			if (workspaceRoot) {
-				await rm(workspaceRoot, { recursive: true, force: true })
-			}
-		},
-		workspaceDir: workspaceRoot ?? null
-	}
-}

ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts

@@ -1,106 +0,0 @@
-import { mkdtemp } from 'fs/promises'
-import { tmpdir } from 'os'
-import { join } from 'path'
-import type { AIProvider, AIProviderModel, ScriptLang } from '$lib/gen/types.gen'
-import type { ContextElement } from '../../../../../frontend/src/lib/components/copilot/chat/context'
-import {
-	prepareScriptSystemMessage,
-	prepareScriptTools,
-	prepareScriptUserMessage,
-	type ScriptChatHelpers
-} from '../../../../../frontend/src/lib/components/copilot/chat/script/core'
-import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers'
-import { runEval } from '../shared'
-import type { ModeRunContext } from '../../../../core/types'
-
-export interface ScriptEvalResult {
-	success: boolean
-	script: ScriptEvalState
-	error?: string
-	assistantMessageCount: number
-	toolCallCount: number
-	toolsUsed: string[]
-}
-
-export interface ScriptEvalOptions {
-	initialScript: ScriptEvalState
-	model?: string
-	maxIterations?: number
-	provider?: AIProvider
-	workspaceRoot?: string
-	runContext?: ModeRunContext
-}
-
-function resolveModelProvider(
-	model: string,
-	provider?: AIProvider
-): AIProviderModel {
-	if (provider) {
-		return { provider, model }
-	}
-	if (model.startsWith('claude')) {
-		return { provider: 'anthropic', model }
-	}
-	return { provider: 'openai', model }
-}
-
-export async function runScriptEval(
-	userPrompt: string,
-	apiKey: string,
-	options: ScriptEvalOptions
-): Promise<ScriptEvalResult> {
-	const workspaceRoot =
-		options.workspaceRoot ?? (await mkdtemp(join(tmpdir(), 'wmill-frontend-script-benchmark-')))
-	const { helpers, getScript, cleanup } = await createScriptFileHelpers(
-		options.initialScript,
-		workspaceRoot
-	)
-
-	try {
-		const model = options.model ?? 'claude-haiku-4-5-20251001'
-		const modelProvider = resolveModelProvider(model, options.provider)
-		const selectedContext: ContextElement[] = []
-		const systemMessage = prepareScriptSystemMessage(
-			modelProvider,
-			options.initialScript.lang,
-			{}
-		)
-		const tools = prepareScriptTools(
-			modelProvider,
-			options.initialScript.lang,
-			selectedContext
-		) as ProductionTool<ScriptChatHelpers>[]
-		const userMessage = prepareScriptUserMessage(userPrompt, selectedContext)
-
-		const rawResult = await runEval({
-			userPrompt,
-			systemMessage,
-			userMessage,
-			tools,
-			helpers,
-			apiKey,
-			getOutput: getScript,
-			onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
-			onAssistantToken: options.runContext?.onAssistantChunk,
-			onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
-			options: {
-				maxIterations: options.maxIterations,
-				model,
-				workspace: workspaceRoot,
-				provider: modelProvider.provider
-			}
-		})
-
-		return {
-			script: rawResult.output,
-			success: rawResult.success,
-			error: rawResult.error,
-			assistantMessageCount: rawResult.iterations,
-			toolCallCount: rawResult.toolCallsCount,
-			toolsUsed: rawResult.toolsCalled
-		}
-	} finally {
-		await cleanup()
-	}
-}

ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts

@@ -1,194 +0,0 @@
-import OpenAI from 'openai'
-import Anthropic from '@anthropic-ai/sdk'
-import type {
-	ChatCompletionMessageParam,
-	ChatCompletionSystemMessageParam
-} from 'openai/resources/chat/completions.mjs'
-import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen'
-import type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types'
-import { runChatLoop, type ChatClients } from '../../../../../frontend/src/lib/components/copilot/chat/chatLoop'
-import type {
-	Tool as ProductionTool,
-	ToolCallbacks
-} from '../../../../../frontend/src/lib/components/copilot/chat/shared'
-
-/**
- * Parameters for running a base evaluation.
- */
-export interface RunEvalParams<THelpers, TOutput> {
-	/** The user's prompt/instruction */
-	userPrompt: string
-	/** System message for the LLM */
-	systemMessage: ChatCompletionSystemMessageParam
-	/** User message for the LLM */
-	userMessage: ChatCompletionMessageParam
-	/** Tool definitions for the LLM API (unused — derived from tools) */
-	toolDefs?: unknown
-	/** Full tool implementations for execution */
-	tools: ProductionTool<THelpers>[]
-	/** Domain-specific helpers for tool execution */
-	helpers: THelpers
-	/** API key for the provider */
-	apiKey: string
-	/** Function to get the current output state */
-	getOutput: () => TOutput
-	/** Optional configuration */
-	options?: EvalRunnerOptions
-	onAssistantMessageStart?: () => void
-	onAssistantToken?: (token: string) => void
-	onAssistantMessageEnd?: () => void
-}
-
-/**
- * Creates SDK clients for the given provider.
- */
-function createEvalClients(provider: AIProvider, apiKey: string): ChatClients {
-	if (provider === 'anthropic') {
-		return {
-			openai: new OpenAI({ apiKey: 'unused' }),
-			anthropic: new Anthropic({ apiKey })
-		} as ChatClients
-	}
-	return {
-		openai: new OpenAI({ apiKey }),
-		anthropic: new Anthropic({ apiKey: 'unused' })
-	} as ChatClients
-}
-
-/**
- * Resolves model string to AIProviderModel.
- */
-function resolveModelProvider(
-	model: string,
-	provider?: AIProvider
-): AIProviderModel {
-	if (provider) return { provider, model }
-	if (model.startsWith('claude')) return { provider: 'anthropic', model }
-	if (model.startsWith('gpt') || model.startsWith('o')) return { provider: 'openai', model }
-	return { provider: 'openai', model }
-}
-
-/**
- * Runs a generic evaluation using the shared chat loop (same code path as production).
- * Uses streaming via real provider SDKs instead of OpenRouter non-streaming.
- */
-export async function runEval<THelpers, TOutput>(
-	params: RunEvalParams<THelpers, TOutput>
-): Promise<RawEvalResult<TOutput>> {
-	const {
-		systemMessage,
-		userMessage,
-		tools,
-		helpers,
-		apiKey,
-		getOutput,
-		options,
-		onAssistantMessageStart,
-		onAssistantToken,
-		onAssistantMessageEnd
-	} = params
-	let shouldEmitMessageStart = true
-
-	const model = options?.model ?? 'gpt-4o'
-	const maxIterations = options?.maxIterations ?? 20
-	const workspace = options?.workspace ?? 'test-workspace'
-	const provider = options?.provider
-
-	const modelProvider = resolveModelProvider(model, provider)
-	const clients = createEvalClients(modelProvider.provider, apiKey)
-
-	const messages: ChatCompletionMessageParam[] = [userMessage]
-	let toolCallsCount = 0
-	const toolsCalled: string[] = []
-	const toolCallDetails: ToolCallDetail[] = []
-
-	// Wrap tools to intercept fn calls for tracking.
-	// Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type
-	// but the actual callbacks passed at runtime will satisfy both interfaces.
-	const wrappedTools = tools.map((tool) => ({
-		...tool,
-		fn: async (p: any) => {
-			toolCallsCount++
-			toolsCalled.push(tool.def.function.name)
-			try {
-				const args =
-					typeof p.args === 'string' ? JSON.parse(p.args) : p.args
-				toolCallDetails.push({ name: tool.def.function.name, arguments: args })
-			} catch {
-				toolCallDetails.push({
-					name: tool.def.function.name,
-					arguments: p.args
-				})
-			}
-			return tool.fn(p)
-		}
-	}))
-
-	// No-op callbacks for eval
-	const callbacks: ToolCallbacks & {
-		onNewToken: (token: string) => void
-		onMessageEnd: () => void
-	} = {
-		setToolStatus: () => {},
-		removeToolStatus: () => {},
-		onNewToken: (token: string) => {
-			if (shouldEmitMessageStart) {
-				onAssistantMessageStart?.()
-				shouldEmitMessageStart = false
-			}
-			onAssistantToken?.(token)
-		},
-		onMessageEnd: () => {
-			onAssistantMessageEnd?.()
-			shouldEmitMessageStart = true
-		}
-	}
-
-	const abortController = new AbortController()
-
-	try {
-		const result = await runChatLoop({
-			messages,
-			systemMessage,
-			tools: wrappedTools,
-			helpers,
-			abortController,
-			callbacks,
-			modelProvider,
-			clients,
-			workspace,
-			maxIterations,
-			skipResponsesApi: modelProvider.provider !== 'openai' && modelProvider.provider !== 'azure_openai'
-		})
-
-		return {
-			success: true,
-			output: getOutput(),
-			tokenUsage: { prompt: 0, completion: 0, total: 0 },
-			toolCallsCount,
-			toolsCalled,
-			toolCallDetails,
-			iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length),
-			messages
-		}
-	} catch (err) {
-		let errorMessage: string
-		if (err instanceof Error) {
-			errorMessage = err.stack ?? err.message
-		} else {
-			errorMessage = String(err)
-		}
-
-		return {
-			success: false,
-			output: getOutput(),
-			error: errorMessage,
-			tokenUsage: { prompt: 0, completion: 0, total: 0 },
-			toolCallsCount,
-			toolsCalled,
-			toolCallDetails,
-			iterations: 0,
-			messages
-		}
-	}
-}

ai_evals/adapters/frontend/core/shared/index.ts

@@ -1,3 +0,0 @@
-export type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types'
-export type { RunEvalParams } from './baseEvalRunner'
-export { runEval } from './baseEvalRunner'

ai_evals/adapters/frontend/core/shared/types.ts

@@ -1,32 +0,0 @@
-import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs'
-import type { AIProvider } from '$lib/gen/types.gen'
-
-export interface TokenUsage {
-	prompt: number
-	completion: number
-	total: number
-}
-
-export interface ToolCallDetail {
-	name: string
-	arguments: Record<string, unknown>
-}
-
-export interface EvalRunnerOptions {
-	maxIterations?: number
-	model?: string
-	workspace?: string
-	provider?: AIProvider
-}
-
-export interface RawEvalResult<TOutput> {
-	success: boolean
-	output: TOutput
-	error?: string
-	tokenUsage: TokenUsage
-	toolCallsCount: number
-	toolsCalled: string[]
-	toolCallDetails: ToolCallDetail[]
-	iterations: number
-	messages: ChatCompletionMessageParam[]
-}

ai_evals/adapters/frontend/progress.ts

@@ -1,133 +0,0 @@
-export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script'
-
-export type FrontendBenchmarkProgressEvent =
-	| {
-			type: 'run-start'
-			surface: FrontendBenchmarkProgressSurface
-			totalCases: number
-			runs: number
-			concurrency: number
-	  }
-	| {
-			type: 'attempt-start'
-			surface: FrontendBenchmarkProgressSurface
-			caseId: string
-			caseNumber: number
-			totalCases: number
-			attempt: number
-			runs: number
-	  }
-	| {
-			type: 'attempt-finish'
-			surface: FrontendBenchmarkProgressSurface
-			caseId: string
-			caseNumber: number
-			totalCases: number
-			attempt: number
-			runs: number
-			passed: boolean
-			durationMs: number
-			judgeScore: number | null
-			error: string | null
-	  }
-	| {
-			type: 'assistant-message-start'
-			surface: FrontendBenchmarkProgressSurface
-			caseId: string
-			caseNumber: number
-			totalCases: number
-			attempt: number
-			runs: number
-	  }
-	| {
-			type: 'assistant-chunk'
-			surface: FrontendBenchmarkProgressSurface
-			caseId: string
-			caseNumber: number
-			totalCases: number
-			attempt: number
-			runs: number
-			chunk: string
-	  }
-	| {
-			type: 'assistant-message-end'
-			surface: FrontendBenchmarkProgressSurface
-			caseId: string
-			caseNumber: number
-			totalCases: number
-			attempt: number
-			runs: number
-	  }
-
-export const FRONTEND_BENCHMARK_PROGRESS_PREFIX = 'WMILL_FRONTEND_AI_EVAL_PROGRESS '
-
-export function emitFrontendBenchmarkProgress(event: FrontendBenchmarkProgressEvent): void {
-	process.stderr.write(
-		`${FRONTEND_BENCHMARK_PROGRESS_PREFIX}${JSON.stringify(event)}\n`
-	)
-}
-
-export function parseFrontendBenchmarkProgressLine(
-	line: string
-): FrontendBenchmarkProgressEvent | null {
-	if (!line.startsWith(FRONTEND_BENCHMARK_PROGRESS_PREFIX)) {
-		return null
-	}
-
-	try {
-		const parsed = JSON.parse(
-			line.slice(FRONTEND_BENCHMARK_PROGRESS_PREFIX.length)
-		) as FrontendBenchmarkProgressEvent
-		return parsed?.type ? parsed : null
-	} catch {
-		return null
-	}
-}
-
-export function formatFrontendBenchmarkProgressEvent(
-	event: FrontendBenchmarkProgressEvent
-): string {
-	switch (event.type) {
-		case 'run-start':
-			return `Running ${event.surface}: ${event.totalCases} cases x ${event.runs} run${event.runs === 1 ? '' : 's'}, concurrency ${event.concurrency}`
-		case 'attempt-start':
-			return `${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs}...`
-		case 'attempt-finish': {
-			const parts = [
-				`${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs} ${event.passed ? 'pass' : 'fail'}`,
-				formatDuration(event.durationMs)
-			]
-			if (event.judgeScore !== null) {
-				parts.push(`judge ${formatNumber(event.judgeScore)}`)
-			}
-			if (event.error) {
-				parts.push(truncateSingleLine(event.error, 120))
-			}
-			return parts.join(' | ')
-		}
-		case 'assistant-message-start':
-		case 'assistant-chunk':
-		case 'assistant-message-end':
-			return ''
-	}
-}
-
-function formatCasePrefix(caseNumber: number, totalCases: number): string {
-	return `[${caseNumber}/${totalCases}]`
-}
-
-function formatDuration(durationMs: number): string {
-	return `${formatNumber(durationMs / 1000)}s`
-}
-
-function formatNumber(value: number): string {
-	return Number.isInteger(value) ? String(value) : value.toFixed(1)
-}
-
-function truncateSingleLine(value: string, maxLength: number): string {
-	const normalized = value.replace(/\s+/g, ' ').trim()
-	if (normalized.length <= maxLength) {
-		return normalized
-	}
-	return `${normalized.slice(0, Math.max(0, maxLength - 3))}...`
-}

ai_evals/adapters/frontend/runtime.ts

@@ -1,215 +0,0 @@
-import { spawn } from 'node:child_process'
-import { mkdtemp, readFile, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import path from 'node:path'
-import { fileURLToPath } from 'node:url'
-import {
-	formatFrontendBenchmarkProgressEvent,
-	parseFrontendBenchmarkProgressLine
-} from './progress'
-import type { BenchmarkRunResult } from '../../core/types'
-
-const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url))
-const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend')
-const FRONTEND_BENCHMARK_TEST = '../ai_evals/adapters/frontend/vitestAdapter.test.ts'
-
-export type FrontendMode = 'flow' | 'app' | 'script'
-
-export async function runFrontendBenchmarkAdapter(input: {
-	mode: FrontendMode
-	caseIds: string[]
-	runs: number
-	model?: string
-	verbose?: boolean
-}): Promise<BenchmarkRunResult> {
-	const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
-	const outputPath = path.join(tempDir, 'result.json')
-
-	try {
-		await runVitestBenchmark(
-			path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'),
-			[
-				'run',
-				FRONTEND_BENCHMARK_TEST,
-				'--project',
-				'server',
-				'--config',
-				'vite.config.js'
-			],
-			{
-				cwd: FRONTEND_DIR,
-				env: {
-					...process.env,
-					BROWSERSLIST_IGNORE_OLD_DATA: '1',
-					WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
-					WMILL_FRONTEND_AI_EVAL_MODE: input.mode,
-					WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
-					WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
-					WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "",
-					WMILL_FRONTEND_AI_EVAL_PROGRESS: '1',
-					WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0'
-				}
-			}
-		)
-
-		const raw = await readFile(outputPath, 'utf8')
-		return JSON.parse(raw) as BenchmarkRunResult
-	} catch (error) {
-		throw new Error(`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`)
-	} finally {
-		await rm(tempDir, { recursive: true, force: true })
-	}
-}
-
-async function runVitestBenchmark(
-	command: string,
-	args: string[],
-	options: {
-		cwd: string
-		env: NodeJS.ProcessEnv
-	}
-): Promise<void> {
-	const child = spawn(command, args, {
-		cwd: options.cwd,
-		env: options.env,
-		stdio: ['ignore', 'pipe', 'pipe']
-	})
-
-	let stdout = ''
-	let stderr = ''
-	let stderrLineBuffer = ''
-	let assistantStreamOpen = false
-
-	child.stdout?.setEncoding('utf8')
-	child.stdout?.on('data', (chunk: string) => {
-		stdout += chunk
-	})
-
-	child.stderr?.setEncoding('utf8')
-	child.stderr?.on('data', (chunk: string) => {
-		stderrLineBuffer += chunk
-		const { remainder, passthrough, nextAssistantStreamOpen } = drainProgressLines(
-			stderrLineBuffer,
-			assistantStreamOpen
-		)
-		stderrLineBuffer = remainder
-		stderr += passthrough
-		assistantStreamOpen = nextAssistantStreamOpen
-	})
-
-	await new Promise<void>((resolve, reject) => {
-		child.once('error', reject)
-		child.once('close', (code) => {
-			if (stderrLineBuffer.length > 0) {
-				const {
-					remainder,
-					passthrough,
-					nextAssistantStreamOpen
-				} = drainProgressLines(`${stderrLineBuffer}\n`, assistantStreamOpen)
-				stderrLineBuffer = remainder
-				stderr += passthrough
-				assistantStreamOpen = nextAssistantStreamOpen
-			}
-
-			if (code === 0) {
-				if (assistantStreamOpen) {
-					process.stderr.write('\n')
-				}
-				resolve()
-				return
-			}
-
-			const details = [`vitest exited with code ${code}`, stdout, stderr].filter(Boolean).join('\n')
-			reject(new Error(details))
-		})
-	})
-}
-
-function drainProgressLines(buffer: string): {
-	remainder: string
-	passthrough: string
-	nextAssistantStreamOpen: boolean
-}
-function drainProgressLines(
-	buffer: string,
-	initialAssistantStreamOpen: boolean
-): {
-	remainder: string
-	passthrough: string
-	nextAssistantStreamOpen: boolean
-} {
-	let remainder = buffer
-	let passthrough = ''
-	let assistantStreamOpen = initialAssistantStreamOpen
-
-	while (true) {
-		const newlineIndex = remainder.indexOf('\n')
-		if (newlineIndex === -1) {
-			return { remainder, passthrough, nextAssistantStreamOpen: assistantStreamOpen }
-		}
-
-		const line = remainder.slice(0, newlineIndex).replace(/\r$/, '')
-		remainder = remainder.slice(newlineIndex + 1)
-
-		const progressEvent = parseFrontendBenchmarkProgressLine(line)
-		if (progressEvent) {
-			if (progressEvent.type === 'assistant-message-start') {
-				if (assistantStreamOpen) {
-					process.stderr.write('\n')
-				}
-				process.stderr.write(
-					`${formatCasePrefix(progressEvent.caseNumber, progressEvent.totalCases)} ${progressEvent.caseId} attempt ${progressEvent.attempt}/${progressEvent.runs} assistant:\n`
-				)
-				assistantStreamOpen = true
-				continue
-			}
-
-			if (progressEvent.type === 'assistant-chunk') {
-				process.stderr.write(progressEvent.chunk)
-				continue
-			}
-
-			if (progressEvent.type === 'assistant-message-end') {
-				if (assistantStreamOpen) {
-					process.stderr.write('\n')
-				}
-				assistantStreamOpen = false
-				continue
-			}
-
-			if (assistantStreamOpen) {
-				process.stderr.write('\n')
-				assistantStreamOpen = false
-			}
-			process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`)
-			continue
-		}
-
-		if (shouldSuppressFrontendStderrLine(line)) {
-			continue
-		}
-
-		passthrough += `${line}\n`
-		process.stderr.write(`${line}\n`)
-	}
-}
-
-function formatCasePrefix(caseNumber: number, totalCases: number): string {
-	return `[${caseNumber}/${totalCases}]`
-}
-
-function shouldSuppressFrontendStderrLine(line: string): boolean {
-	return (
-		line.startsWith('[baseline-browser-mapping] ') ||
-		line.startsWith('Browserslist: browsers data (caniuse-lite) is ') ||
-		line.includes('update-browserslist-db@latest') ||
-		line.includes('update-db#readme')
-	)
-}
-
-function toErrorMessage(error: unknown): string {
-	if (error instanceof Error) {
-		return error.message
-	}
-	return String(error)
-}

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -1,48 +0,0 @@
-import { expect, it, vi } from 'vitest'
-// @ts-ignore - Node.js fs/promises
-import { mkdir, writeFile } from 'fs/promises'
-// @ts-ignore - Node.js path
-import { dirname, resolve } from 'path'
-
-vi.mock('monaco-editor', () => ({
-	editor: {},
-	languages: {},
-	KeyCode: {},
-	Uri: {
-		parse: (value: string) => ({ toString: () => value })
-	},
-	MarkerSeverity: {
-		Error: 8,
-		Warning: 4,
-		Info: 2,
-		Hint: 1
-	}
-}))
-
-vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({
-	getTypeScriptWorker: async () => async () => ({}),
-	typescriptVersion: 'test'
-}))
-
-vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({
-	default: () => ({})
-}))
-
-vi.mock('$lib/components/vscode', () => ({}))
-
-const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH
-const benchmarkIt = benchmarkOutputPath ? it : it.skip
-
-benchmarkIt(
-	'runs the frontend benchmark adapter from environment input',
-	async () => {
-		const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner')
-		const payload = await runFrontendBenchmarkFromEnv()
-		const absoluteOutputPath = resolve(benchmarkOutputPath!)
-		await mkdir(dirname(absoluteOutputPath), { recursive: true })
-		await writeFile(absoluteOutputPath, JSON.stringify(payload, null, 2) + '\n', 'utf8')
-
-		expect(payload.cases.length).toBeGreaterThan(0)
-	},
-	600_000
-)

ai_evals/bun.lock

@@ -1,310 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "configVersion": 1,
-  "workspaces": {
-    "": {
-      "name": "windmill-ai-evals",
-      "dependencies": {
-        "@anthropic-ai/claude-agent-sdk": "^0.2.25",
-        "@anthropic-ai/sdk": "^0.39.0",
-        "commander": "^14.0.3",
-        "yaml": "^2.8.3",
-      },
-      "devDependencies": {
-        "@types/bun": "latest",
-        "typescript": "^5.0.0",
-      },
-    },
-  },
-  "packages": {
-    "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="],
-
-    "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="],
-
-    "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
-
-    "@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
-
-    "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
-
-    "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
-
-    "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
-
-    "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
-
-    "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
-
-    "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
-
-    "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
-
-    "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
-
-    "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
-
-    "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
-
-    "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
-
-    "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
-
-    "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
-
-    "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
-
-    "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
-
-    "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
-
-    "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
-
-    "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
-
-    "@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
-
-    "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
-
-    "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
-
-    "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
-
-    "agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
-
-    "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
-
-    "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
-
-    "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
-
-    "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
-
-    "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
-
-    "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
-
-    "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
-
-    "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
-
-    "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
-
-    "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="],
-
-    "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
-
-    "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="],
-
-    "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
-
-    "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
-
-    "cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="],
-
-    "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
-
-    "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
-
-    "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
-
-    "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
-
-    "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
-
-    "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],
-
-    "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
-
-    "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
-
-    "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
-
-    "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
-
-    "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
-
-    "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
-
-    "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
-
-    "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
-
-    "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
-
-    "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
-
-    "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="],
-
-    "express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
-
-    "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
-
-    "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
-
-    "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
-
-    "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
-
-    "form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
-
-    "formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="],
-
-    "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
-
-    "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
-
-    "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
-
-    "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
-
-    "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
-
-    "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
-
-    "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
-
-    "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
-
-    "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
-
-    "hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="],
-
-    "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
-
-    "humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="],
-
-    "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
-
-    "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
-
-    "ip-address": ["ip-address@10.1.0", "", {}, "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q=="],
-
-    "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
-
-    "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
-
-    "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
-
-    "jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="],
-
-    "json-schema-to-ts": ["json-schema-to-ts@3.1.1", "", { "dependencies": { "@babel/runtime": "^7.18.3", "ts-algebra": "^2.0.0" } }, "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g=="],
-
-    "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
-
-    "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="],
-
-    "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
-
-    "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
-
-    "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="],
-
-    "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
-
-    "mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="],
-
-    "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
-
-    "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
-
-    "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
-
-    "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
-
-    "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
-
-    "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
-
-    "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="],
-
-    "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
-
-    "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
-
-    "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
-
-    "path-to-regexp": ["path-to-regexp@8.4.1", "", {}, "sha512-fvU78fIjZ+SBM9YwCknCvKOUKkLVqtWDVctl0s7xIqfmfb38t2TT4ZU2gHm+Z8xGwgW+QWEU3oQSAzIbo89Ggw=="],
-
-    "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="],
-
-    "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
-
-    "qs": ["qs@6.15.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ=="],
-
-    "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
-
-    "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="],
-
-    "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
-
-    "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="],
-
-    "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
-
-    "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="],
-
-    "serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="],
-
-    "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
-
-    "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
-
-    "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
-
-    "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
-
-    "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="],
-
-    "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="],
-
-    "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
-
-    "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
-
-    "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
-
-    "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
-
-    "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="],
-
-    "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
-
-    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
-
-    "undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
-
-    "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
-
-    "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
-
-    "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
-
-    "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
-
-    "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
-
-    "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
-
-    "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
-
-    "yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="],
-
-    "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
-
-    "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
-
-    "@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="],
-
-    "@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
-
-    "bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
-
-    "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
-
-    "@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
-
-    "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
-
-    "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
-  }
-}

ai_evals/cases/app.yaml

@@ -1,46 +0,0 @@
-- id: app-test1-counter-create
-  prompt: |-
-    Create a counter app with increment/decrement buttons
-
-- id: app-test2-counter-reset
-  prompt: |-
-    Add a reset button that sets the counter back to 0
-  initial: ai_evals/fixtures/frontend/app/initial/test1_counter_app
-
-- id: app-test3-shopping-cart-quantity
-  prompt: |-
-    Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items
-  initial: ai_evals/fixtures/frontend/app/initial/shopping_cart
-
-- id: app-test4-shopping-cart-discount
-  prompt: |-
-    Add a discount code input field in the cart.
-    When the code "SAVE10" is entered, apply a 10% discount to the total
-  initial: ai_evals/fixtures/frontend/app/initial/shopping_cart
-
-- id: app-test5-file-manager-search
-  prompt: |-
-    Add a search bar in the toolbar that filters files and folders by name as the user types
-  initial: ai_evals/fixtures/frontend/app/initial/file_manager
-
-- id: app-test6-file-manager-details
-  prompt: |-
-    Show file size (formatted as KB/MB) and modified date in the file list for each item
-  initial: ai_evals/fixtures/frontend/app/initial/file_manager
-
-- id: app-test7-file-manager-select-all
-  prompt: |-
-    Add a "Select All" checkbox in the file list header and individual checkboxes for each file.
-    Add a "Delete Selected" button that appears when items are selected
-  initial: ai_evals/fixtures/frontend/app/initial/file_manager
-
-- id: app-test8-quiz-create
-  prompt: |-
-    Create a multiple choice quiz app with 5 questions about general knowledge.
-    Show one question at a time with 4 answer options.
-    Track the score and show results at the end with percentage correct.
-
-- id: app-test9-recipe-book-create
-  prompt: |-
-    Create a recipe book app where users can add recipes with a name, ingredients list, and instructions.
-    Include a search bar to filter recipes by name and the ability to delete recipes.

ai_evals/cases/cli.yaml

@@ -1,21 +0,0 @@
-- id: bun-hello-script
-  prompt: |-
-    This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts.
-    The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }.
-    Keep it minimal.
-    Do not create other scripts.
-    Do not run any CLI commands.
-    After writing the file, tell me exactly which wmill commands I should run next.
-  expected: ai_evals/fixtures/cli/expected/bun-hello-script
-
-- id: bun-hello-flow
-  prompt: |-
-    This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow.
-    The flow must contain flow.yaml and one inline Bun script file named hello.ts.
-    The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }.
-    Use a single rawscript step wired to that input.
-    Keep it minimal.
-    Do not create any other flows or scripts.
-    Do not run any CLI commands.
-    After writing the files, tell me exactly which wmill commands I should run next.
-  expected: ai_evals/fixtures/cli/expected/bun-hello-flow

ai_evals/cases/flow.yaml

@@ -1,246 +0,0 @@
-- id: flow-test0-sum-two-numbers
-  prompt: |-
-    Create a flow that takes two numbers, `a` and `b`, and returns their sum.
-    Keep it simple and use a single step named `sum_numbers`.
-  expected: ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json
-  judgeChecklist:
-    - "the flow takes `a` and `b` as inputs"
-    - "the main step is named `sum_numbers`"
-    - the flow returns the sum of the two numbers
-
-- id: flow-test1-reuse-existing-script
-  prompt: |-
-    I need a flow that adds two numbers.
-    If there is already a script in the workspace that does that, reuse it instead of rewriting the logic.
-    The flow should take `a` and `b` as inputs and use a single step named `sum_numbers`.
-  initial: ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json
-  expected: ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json
-  judgeChecklist:
-    - "the flow takes `a` and `b` as inputs"
-    - "the main step is named `sum_numbers`"
-    - the flow reuses the existing workspace script instead of rewriting the addition logic
-
-- id: flow-test2-call-existing-subflow
-  prompt: |-
-    Create a parent flow that adds two numbers by reusing an existing flow in the workspace if one already exists.
-    The parent flow should take `a` and `b` as inputs and delegate the calculation instead of inlining it.
-    Use a single step named `call_add_numbers`.
-  initial: ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json
-  expected: ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json
-  judgeChecklist:
-    - "the parent flow takes `a` and `b` as inputs"
-    - "the main step is named `call_add_numbers`"
-    - the parent flow delegates to an existing workspace subflow instead of inlining the addition logic
-
-- id: flow-test3-branchone-routing
-  prompt: |-
-    Create a flow that routes incoming support requests based on the customer's tier.
-    The input should contain a string field named `tier`.
-    Free, pro, and enterprise requests should go to different queues, and unknown tiers should fall back to a default queue.
-    Name the main routing step `route_by_tier`.
-  expected: ai_evals/fixtures/frontend/flow/expected/test3_branchone_routing.json
-  judgeChecklist:
-    - "the input schema includes a string field named `tier`"
-    - "the main routing step is named `route_by_tier`"
-    - free requests go to a free queue
-    - pro requests go to a pro queue
-    - enterprise requests go to an enterprise queue
-    - unknown tiers fall back to a default queue
-
-- id: flow-test4-order-processing-loop
-  prompt: |-
-    Build an order-processing flow.
-
-    The input should include an order with:
-    - an `items` array containing `name`, `price`, and `quantity`
-    - `customer_email`
-    - `shipping_address`
-
-    The flow should:
-    - validate that every item has a positive price and quantity
-    - calculate the order total with 8% tax
-    - check inventory for each item using placeholder availability data
-    - create a shipment if everything is in stock, otherwise create a backorder
-    - send a confirmation using placeholder email logic
-    - return a final order summary with the status
-  validate:
-    schemaAnyOf:
-      - requiredPaths:
-          - order
-          - order.items
-          - order.customer_email
-          - order.shipping_address
-      - requiredPaths:
-          - items
-          - customer_email
-          - shipping_address
-    resolveResultsRefs: true
-  judgeChecklist:
-    - the flow validates that every item has a positive price and quantity
-    - the flow calculates the order total with 8% tax
-    - the flow checks inventory for each item using placeholder availability data
-    - the flow creates a shipment if everything is in stock, otherwise a backorder
-    - the flow sends a confirmation using placeholder email logic
-    - the flow returns a final order summary with the resulting status
-
-- id: flow-test5-parallel-data-pipeline
-  prompt: |-
-    Create a data-processing flow for three external data sources.
-
-    It should:
-    - load a small placeholder configuration listing the three sources
-    - fetch placeholder records from each source
-    - clean and validate each source's records
-    - combine everything into one dataset
-    - compute an overall quality score
-    - store the result differently depending on the score:
-      - 90 or above goes to the primary database
-      - 70 to 89 goes to a secondary database with a warning
-      - below 70 goes to quarantine and triggers an alert
-    - return a processing report with total records, quality score, and destination
-  judgeChecklist:
-    - the flow loads a placeholder configuration listing three external sources
-    - the flow fetches placeholder records from each source
-    - the flow cleans and validates each source's records
-    - the flow combines everything into one dataset
-    - the flow computes an overall quality score
-    - scores of 90 or above go to the primary database
-    - scores from 70 to 89 go to a secondary database with a warning
-    - scores below 70 go to quarantine and trigger an alert
-    - the final report includes total records, quality score, and destination
-
-- id: flow-test6-ai-agent-tools
-  prompt: |-
-    Create a customer support flow.
-
-    The input should include `customer_id` and `query_text`.
-    The flow should load the customer's profile and order history, then use an AI assistant to help with the request.
-    The assistant should be able to:
-    - look up orders
-    - check refund eligibility
-    - search FAQs
-    - open a support ticket when needed
-
-    After that, log the interaction and return the assistant's response along with any actions it took.
-  judgeChecklist:
-    - "the input schema includes `customer_id` and `query_text`"
-    - the flow loads the customer's profile and order history
-    - the flow uses an AI assistant step
-    - the assistant can look up orders
-    - the assistant can check refund eligibility
-    - the assistant can search FAQs
-    - the assistant can open a support ticket
-    - the flow logs the interaction
-    - the final output returns the assistant response along with any actions taken or resulting support action details
-
-- id: flow-test7-simple-modification
-  prompt: |-
-    Update this flow so it validates processed data before saving it.
-
-    After `process_data`, add a `validate_data` step that checks the data array is not empty.
-    If the array is empty, it should return an error object with the message `No data to save`.
-    If validation passes, let the save continue normally.
-    Update `save_results` so it handles the validation result correctly.
-  initial: ai_evals/fixtures/frontend/flow/initial/test5_initial.json
-  expected: ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json
-  judgeChecklist:
-    - the updated flow keeps the original fetch and process steps intact
-    - "a `validate_data` step is added after `process_data`"
-    - "`validate_data` checks that the processed data array is not empty"
-    - "empty data returns an error object with the message `No data to save`"
-    - "`save_results` handles the validation result correctly"
-
-- id: flow-test8-branching-in-loop
-  prompt: |-
-    Update the order-processing logic inside `loop_orders` so different order types are handled differently.
-
-    For `express`, mark the order as priority and use a shipping cost of $15.99.
-    For `standard`, use a shipping cost of $5.99.
-    For `pickup`, mark it as no shipping required with a cost of $0.
-    Keep the existing processing as a fallback for unknown order types.
-    Each path should return the orderId, shipping cost, and shipping type.
-  initial: ai_evals/fixtures/frontend/flow/initial/test6_initial.json
-  judgeChecklist:
-    - "the existing `loop_orders` flow still handles per-order processing"
-    - exact branching topology is not required as long as `loop_orders` handles the order types correctly
-    - express orders are marked as priority and use a shipping cost of 15.99
-    - standard orders use a shipping cost of 5.99
-    - pickup orders use a shipping cost of 0 and are treated as no shipping required
-    - unknown order types still follow a fallback path
-    - "each processed order returns `orderId`, `shippingCost`, and `shippingType`"
-
-- id: flow-test9-parallel-refactor
-  prompt: |-
-    Refactor this flow so the enrichment work no longer runs one step at a time.
-
-    `enrich_price`, `enrich_inventory`, and `enrich_reviews` should run independently.
-    Each one should return a fallback value if it fails.
-    Update `combine_data` so it merges the enrichment results and sets a `hasFallbacks` flag when any fallback was used.
-    Keep `get_item` as the first step and `return_result` as the last step.
-  initial: ai_evals/fixtures/frontend/flow/initial/test7_initial.json
-  expected: ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json
-  judgeChecklist:
-    - "the updated flow keeps `get_item` as the first step"
-    - "the updated flow keeps `return_result` as the last step"
-    - "`enrich_price`, `enrich_inventory`, and `enrich_reviews` run independently rather than sequentially"
-    - each enrichment path returns a fallback value if it fails
-    - "`combine_data` merges the enrichment results"
-    - "`combine_data` sets `hasFallbacks` when any fallback was used"
-
-- id: flow-test10-while-loop-counter
-  prompt: |-
-    Create a flow that keeps incrementing a counter until it reaches a target value.
-    The input should include a number field named `target`.
-    Name the looping step `count_until_target`.
-    Once the target is reached, return the final counter value.
-  expected: ai_evals/fixtures/frontend/flow/expected/test10_while_loop_counter.json
-  judgeChecklist:
-    - "the input schema includes a number field named `target`"
-    - "the looping step is named `count_until_target`"
-    - the flow keeps incrementing a counter until the target is reached
-    - the final output returns the final counter value
-
-- id: flow-test11-preprocessor-and-failure-handler
-  prompt: |-
-    Create an event-processing flow for a string payload.
-
-    Before the main processing runs, trim the payload and reject empty strings.
-    The main step should be named `process_event` and return a simple success object.
-    If anything fails, return a compact error object with the error message and the failing step id.
-  expected: ai_evals/fixtures/frontend/flow/expected/test11_preprocessor_failure.json
-  validate:
-    requireSpecialModules:
-      - preprocessor_module
-      - failure_module
-  judgeChecklist:
-    - the flow trims the payload before the main processing runs
-    - the flow rejects empty payload strings
-    - "the main step is named `process_event`"
-    - "`process_event` returns a simple success object"
-    - failures return a compact error object with the error message and failing step id
-
-- id: flow-test12-approval-step
-  prompt: |-
-    Create a purchase approval flow.
-
-    The input should include `requester_email` and `amount`.
-    Add an approval step named `request_approval` that pauses the flow and asks the approver for a comment.
-    One approval should be enough to continue.
-    After approval, add a final step named `finalize_purchase` that returns an approved status object.
-  expected: ai_evals/fixtures/frontend/flow/expected/test12_approval_step.json
-  validate:
-    schemaRequiredPaths:
-      - requester_email
-      - amount
-    requireSuspendSteps:
-      - id: request_approval
-        requiredEvents: 1
-        resumeRequiredStringFieldAnyOf:
-          - comment
-          - approver_comment
-  judgeChecklist:
-    - "the flow includes an approval step named `request_approval`"
-    - "`request_approval` pauses the flow and asks the approver for a comment"
-    - one approval is enough to continue
-    - "the flow includes a final step named `finalize_purchase`"
-    - "`finalize_purchase` returns an approved status object after approval"

ai_evals/cases/script.yaml

@@ -1,7 +0,0 @@
-- id: script-test1-greet-user
-  prompt: |-
-    Update the current Bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.
-    Do not return an object or array.
-    Do not add external dependencies.
-  initial: ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json
-  expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json

ai_evals/cli/index.ts

@@ -1,217 +0,0 @@
-#!/usr/bin/env bun
-
-import { Command, InvalidArgumentError } from "commander";
-import { loadCases, loadSelectedCases } from "../core/cases";
-import {
-  EVAL_MODELS,
-  formatRunModelLabel,
-  getCliEvalModel,
-  getEvalModelHelpText,
-  resolveEvalModel,
-} from "../core/models";
-import {
-  appendHistoryRecord,
-  buildRunResult,
-  formatRunSummary,
-  resolveRunOutputPath,
-  writeRunArtifacts,
-  writeRunResult,
-} from "../core/results";
-import { runSuite } from "../core/runSuite";
-import { EVAL_MODES, type EvalMode } from "../core/types";
-import { DEFAULT_JUDGE_MODEL } from "../core/judge";
-import { createCliModeRunner } from "../modes/cli";
-import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime";
-
-async function main() {
-  const program = new Command()
-    .name("bun run cli --")
-    .description("Run AI eval cases against the current production prompts and guidance")
-    .showHelpAfterError()
-    .showSuggestionAfterError()
-    .addHelpText(
-      "after",
-      [
-        "",
-        "Examples:",
-        "  bun run cli -- models",
-        "  bun run cli -- cases",
-        "  bun run cli -- cases flow",
-        "  bun run cli -- run flow",
-        "  bun run cli -- run flow --model 4o",
-        "  bun run cli -- run flow flow-test0-sum-two-numbers --verbose",
-        "  bun run cli -- run flow --record",
-        "  bun run cli -- run flow flow-test5-simple-modification --runs 3",
-        "  bun run cli -- run cli bun-hello-script",
-        "",
-        "Models:",
-        getEvalModelHelpText(),
-      ].join("\n")
-    );
-
-  program
-    .command("models")
-    .description("List available model aliases")
-    .action(() => {
-      handleModels();
-    });
-
-  program
-    .command("cases")
-    .description("List available cases")
-    .argument("[mode]", "cli, flow, script, or app", parseOptionalMode)
-    .action(async (mode?: EvalMode) => {
-      await handleCases(mode);
-    });
-
-  program
-    .command("run")
-    .description("Run one benchmark mode")
-    .argument("<mode>", "cli, flow, script, or app", parseMode)
-    .argument("[caseIds...]", "specific case ids to run")
-    .option("--runs <n>", "number of attempts per case", parsePositiveInteger, 1)
-    .option("--output <path>", "write the result JSON to this path")
-    .option("--model <name>", `model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`)
-    .option("--verbose", "stream assistant output during frontend runs")
-    .option("--record", "append a compact summary line to ai_evals/history/<mode>.jsonl")
-    .action(
-      async (
-        mode: EvalMode,
-        caseIds: string[],
-        options: {
-          runs: number;
-          output?: string;
-          model?: string;
-          verbose?: boolean;
-          record?: boolean;
-        }
-      ) => {
-        await handleRun({
-          mode,
-          caseIds,
-          runs: options.runs,
-          outputPath: options.output,
-          model: options.model,
-          verbose: options.verbose ?? false,
-          record: options.record ?? false,
-        });
-      }
-    );
-
-  await program.parseAsync(process.argv);
-}
-
-async function handleCases(mode?: EvalMode) {
-  const modes = mode ? [mode] : [...EVAL_MODES];
-
-  for (const entry of modes) {
-    const cases = await loadCases(entry);
-    process.stdout.write(`${entry} (${cases.length})\n`);
-    for (const evalCase of cases) {
-      process.stdout.write(`- ${evalCase.id}\n`);
-    }
-    process.stdout.write("\n");
-  }
-}
-
-function handleModels() {
-  process.stdout.write("Available models\n");
-  for (const model of EVAL_MODELS) {
-    const supports = [
-      ...(model.frontend ? ["flow", "script", "app"] : []),
-      ...(model.cli ? ["cli"] : []),
-    ];
-    const aliases = [model.id, ...model.aliases.filter((alias) => alias !== model.id)];
-    process.stdout.write(`- ${model.id}: ${model.label}\n`);
-    process.stdout.write(`  aliases: ${aliases.join(", ")}\n`);
-    process.stdout.write(`  modes: ${supports.join(", ")}\n`);
-  }
-  process.stdout.write(`\nJudge model: ${DEFAULT_JUDGE_MODEL}\n`);
-}
-
-async function handleRun(input: {
-  mode: EvalMode;
-  caseIds: string[];
-  runs: number;
-  outputPath?: string;
-  model?: string;
-  verbose: boolean;
-  record: boolean;
-}) {
-  const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
-  const model = resolveEvalModel(input.mode, input.model);
-  const runModel = formatRunModelLabel(input.mode, model);
-  process.stderr.write(`Starting ${input.mode} benchmark...\n`);
-
-  const result =
-    input.mode === "cli"
-      ? await runCliBenchmark(selectedCases, input.runs, getCliEvalModel(model), runModel)
-      : await runFrontendBenchmarkAdapter({
-          mode: input.mode,
-          caseIds: input.caseIds,
-          runs: input.runs,
-          model: model.id,
-          verbose: input.verbose,
-        });
-
-  const resolvedOutputPath = resolveRunOutputPath(input.mode, input.outputPath);
-  const artifactsPath = await writeRunArtifacts(result, resolvedOutputPath);
-  const resultPath = await writeRunResult(result, resolvedOutputPath);
-  const historyPath = input.record ? await appendHistoryRecord(result) : null;
-  process.stdout.write(`${formatRunSummary(result)}\n`);
-  process.stdout.write(`Saved: ${resultPath}\n`);
-  if (artifactsPath) {
-    process.stdout.write(`Artifacts: ${artifactsPath}\n`);
-  }
-  if (historyPath) {
-    process.stdout.write(`Recorded: ${historyPath}\n`);
-  }
-}
-
-async function runCliBenchmark(
-  cases: Awaited<ReturnType<typeof loadSelectedCases>>,
-  runs: number,
-  model: ReturnType<typeof getCliEvalModel>,
-  runModel: string
-) {
-  const caseResults = await runSuite({
-    modeRunner: createCliModeRunner(model),
-    cases,
-    runs,
-    runModel,
-    judgeModel: DEFAULT_JUDGE_MODEL,
-  });
-
-  return buildRunResult({
-    mode: "cli",
-    runs,
-    runModel,
-    judgeModel: DEFAULT_JUDGE_MODEL,
-    caseResults,
-  });
-}
-
-function parseMode(value: string): EvalMode {
-  if (EVAL_MODES.includes(value as EvalMode)) {
-    return value as EvalMode;
-  }
-  throw new InvalidArgumentError(`mode must be one of: ${EVAL_MODES.join(", ")}`);
-}
-
-function parseOptionalMode(value: string | undefined): EvalMode | undefined {
-  return value ? parseMode(value) : undefined;
-}
-
-function parsePositiveInteger(value: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed <= 0) {
-    throw new InvalidArgumentError("must be a positive integer");
-  }
-  return parsed;
-}
-
-void main().catch((error) => {
-  const message = error instanceof Error ? error.message : String(error);
-  process.stderr.write(`${message}\n`);
-  process.exit(1);
-});

ai_evals/core/cases.ts

@@ -1,67 +0,0 @@
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { parse } from "yaml";
-import type { EvalCase, EvalMode, FlowValidationSpec } from "./types";
-
-const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url));
-const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases");
-
-interface RawEvalCase {
-  id: string;
-  prompt: string;
-  initial?: string;
-  expected?: string;
-  validate?: FlowValidationSpec;
-  judgeChecklist?: string[];
-}
-
-export function getRepoRoot(): string {
-  return REPO_ROOT;
-}
-
-export function getAiEvalsRoot(): string {
-  return path.join(REPO_ROOT, "ai_evals");
-}
-
-export async function loadCases(mode: EvalMode): Promise<EvalCase[]> {
-  const filePath = path.join(CASES_DIR, `${mode}.yaml`);
-  const raw = await readFile(filePath, "utf8");
-  const parsed = parse(raw) as RawEvalCase[];
-
-  return parsed.map((entry) => ({
-    id: entry.id,
-    prompt: entry.prompt,
-    initialPath: resolveFixturePath(entry.initial),
-    expectedPath: resolveFixturePath(entry.expected),
-    validate: entry.validate,
-    judgeChecklist: entry.judgeChecklist,
-  }));
-}
-
-export async function loadSelectedCases(
-  mode: EvalMode,
-  selectedIds: string[]
-): Promise<EvalCase[]> {
-  const allCases = await loadCases(mode);
-  if (selectedIds.length === 0) {
-    return allCases;
-  }
-
-  const caseMap = new Map(allCases.map((entry) => [entry.id, entry]));
-  const missing = selectedIds.filter((id) => !caseMap.has(id));
-  if (missing.length > 0) {
-    throw new Error(
-      `Unknown ${mode} case${missing.length === 1 ? "" : "s"}: ${missing.join(", ")}`
-    );
-  }
-
-  return selectedIds.map((id) => caseMap.get(id)!);
-}
-
-function resolveFixturePath(value: string | undefined): string | undefined {
-  if (!value) {
-    return undefined;
-  }
-  return path.isAbsolute(value) ? value : path.join(REPO_ROOT, value);
-}

ai_evals/core/files.ts

@@ -1,67 +0,0 @@
-import { access, copyFile, mkdir, readdir, readFile } from "node:fs/promises";
-import path from "node:path";
-
-export async function exists(filePath: string): Promise<boolean> {
-  try {
-    await access(filePath);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-export async function readJsonFile<T>(filePath: string): Promise<T> {
-  const raw = await readFile(filePath, "utf8");
-  return JSON.parse(raw) as T;
-}
-
-export async function readDirectoryFiles(
-  rootDir: string,
-  options: {
-    ignore?: Set<string>;
-  } = {}
-): Promise<Record<string, string>> {
-  const files: Record<string, string> = {};
-  await walkDirectory(rootDir, "", files, options.ignore ?? new Set());
-  return files;
-}
-
-export async function copyDirectory(sourceDir: string, targetDir: string): Promise<void> {
-  const entries = await readdir(sourceDir, { withFileTypes: true });
-  await mkdir(targetDir, { recursive: true });
-
-  for (const entry of entries) {
-    const sourcePath = path.join(sourceDir, entry.name);
-    const targetPath = path.join(targetDir, entry.name);
-    if (entry.isDirectory()) {
-      await copyDirectory(sourcePath, targetPath);
-      continue;
-    }
-    await mkdir(path.dirname(targetPath), { recursive: true });
-    await copyFile(sourcePath, targetPath);
-  }
-}
-
-async function walkDirectory(
-  absoluteDir: string,
-  relativeDir: string,
-  output: Record<string, string>,
-  ignore: Set<string>
-): Promise<void> {
-  const entries = await readdir(absoluteDir, { withFileTypes: true });
-
-  for (const entry of entries) {
-    const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
-    if (ignore.has(relativePath) || ignore.has(entry.name)) {
-      continue;
-    }
-
-    const absolutePath = path.join(absoluteDir, entry.name);
-    if (entry.isDirectory()) {
-      await walkDirectory(absolutePath, relativePath, output, ignore);
-      continue;
-    }
-
-    output[relativePath] = await readFile(absolutePath, "utf8");
-  }
-}

ai_evals/core/judge.ts

@@ -1,149 +0,0 @@
-import Anthropic from "@anthropic-ai/sdk";
-import type { EvalMode, JudgeResult } from "./types";
-
-export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6";
-
-const JUDGE_TOOL_NAME = "submit_judgement";
-
-export async function judgeOutput(input: {
-  mode: EvalMode;
-  prompt: string;
-  checklist?: string[];
-  initial?: unknown;
-  expected?: unknown;
-  actual: unknown;
-  model?: string;
-}): Promise<JudgeResult> {
-  const apiKey = process.env.ANTHROPIC_API_KEY;
-  if (!apiKey) {
-    return {
-      success: false,
-      score: 0,
-      summary: "Judge unavailable",
-      error: "ANTHROPIC_API_KEY is not set",
-    };
-  }
-
-  const client = new Anthropic({ apiKey });
-  const model = input.model ?? DEFAULT_JUDGE_MODEL;
-
-  const system = [
-    "You evaluate benchmark outputs for Windmill AI generation.",
-    "Deterministic checks already run separately. Focus on whether the final output satisfies the user request.",
-    "If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.",
-    "If a checklist is provided, treat it as the explicit acceptance criteria for this case.",
-    "Be strict about missing requested functionality.",
-    "When the prompt wording is ambiguous, prefer the checklist over inferred structural requirements.",
-    "Do not require exact ids, exact topology, or exact field names unless the prompt, checklist, or expected state clearly requires them.",
-    `Always respond by calling the ${JUDGE_TOOL_NAME} tool exactly once.`,
-  ].join("\n\n");
-
-  const user = [
-    `Mode: ${input.mode}`,
-    "",
-    "User prompt:",
-    input.prompt,
-    "",
-    "Checklist:",
-    formatChecklist(input.checklist),
-    "",
-    "Initial state:",
-    formatJsonBlock(input.initial),
-    "",
-    "Expected state:",
-    formatJsonBlock(input.expected),
-    "",
-    "Actual result:",
-    formatJsonBlock(input.actual),
-  ].join("\n");
-
-  try {
-    const response = await client.messages.create({
-      model,
-      max_tokens: 1024,
-      temperature: 0,
-      system,
-      messages: [{ role: "user", content: user }],
-      tools: [
-        {
-          name: JUDGE_TOOL_NAME,
-          description: "Submit the benchmark judgement as structured data.",
-          input_schema: {
-            type: "object",
-            properties: {
-              score: {
-                type: "integer",
-                minimum: 0,
-                maximum: 100,
-              },
-              summary: {
-                type: "string",
-              },
-            },
-            required: ["score", "summary"],
-          },
-        },
-      ],
-      tool_choice: {
-        type: "tool",
-        name: JUDGE_TOOL_NAME,
-        disable_parallel_tool_use: true,
-      },
-    });
-
-    const toolUseBlock = response.content.find(
-      (block): block is Anthropic.ToolUseBlock =>
-        block.type === "tool_use" && block.name === JUDGE_TOOL_NAME
-    );
-
-    if (!toolUseBlock) {
-      return {
-        success: false,
-        score: 0,
-        summary: "Judge returned no tool output",
-        error: "Expected structured tool output from judge",
-      };
-    }
-
-    const parsed = toolUseBlock.input as {
-      score: number;
-      summary: string;
-    };
-
-    return {
-      success: true,
-      score: normalizeScore(parsed.score),
-      summary: parsed.summary,
-    };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return {
-      success: false,
-      score: 0,
-      summary: "Judge failed",
-      error: message,
-    };
-  }
-}
-
-function formatJsonBlock(value: unknown): string {
-  if (value === undefined) {
-    return "(none)";
-  }
-  return JSON.stringify(value, null, 2);
-}
-
-function formatChecklist(checklist: string[] | undefined): string {
-  if (!checklist || checklist.length === 0) {
-    return "(none)";
-  }
-
-  return checklist.map((item) => `- ${item}`).join("\n");
-}
-
-function normalizeScore(value: number): number {
-  if (!Number.isFinite(value)) {
-    return 0;
-  }
-  return Math.max(0, Math.min(100, Math.round(value)));
-}

ai_evals/core/models.ts

@@ -1,149 +0,0 @@
-import type { EvalMode } from "./types";
-
-export interface FrontendEvalModelConfig {
-  provider: "anthropic" | "openai";
-  model: string;
-}
-
-export interface CliEvalModelConfig {
-  provider: "anthropic";
-  model: string;
-}
-
-export interface EvalModelSpec {
-  id: string;
-  label: string;
-  aliases: string[];
-  frontend?: FrontendEvalModelConfig;
-  cli?: CliEvalModelConfig;
-}
-
-export const EVAL_MODELS: EvalModelSpec[] = [
-  {
-    id: "haiku",
-    label: "Claude Haiku 4.5",
-    aliases: [
-      "haiku",
-      "haiku-4.5",
-      "claude-haiku",
-      "claude-haiku-4.5",
-      "claude-haiku-4-5",
-      "claude-haiku-4-5-20251001",
-    ],
-    frontend: {
-      provider: "anthropic",
-      model: "claude-haiku-4-5-20251001",
-    },
-    cli: {
-      provider: "anthropic",
-      model: "haiku",
-    },
-  },
-  {
-    id: "sonnet",
-    label: "Claude Sonnet 4.5",
-    aliases: [
-      "sonnet",
-      "sonnet-4.5",
-      "claude-sonnet",
-      "claude-sonnet-4.5",
-      "claude-sonnet-4-5",
-      "claude-sonnet-4-5-20250929",
-    ],
-    frontend: {
-      provider: "anthropic",
-      model: "claude-sonnet-4-5-20250929",
-    },
-    cli: {
-      provider: "anthropic",
-      model: "sonnet",
-    },
-  },
-  {
-    id: "opus",
-    label: "Claude Opus 4.6",
-    aliases: [
-      "opus",
-      "opus-4.6",
-      "claude-opus",
-      "claude-opus-4.6",
-      "claude-opus-4-6",
-    ],
-    frontend: {
-      provider: "anthropic",
-      model: "claude-opus-4-6",
-    },
-    cli: {
-      provider: "anthropic",
-      model: "opus",
-    },
-  },
-  {
-    id: "4o",
-    label: "GPT-4o",
-    aliases: ["4o", "gpt-4o"],
-    frontend: {
-      provider: "openai",
-      model: "gpt-4o",
-    },
-  },
-];
-
-export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec {
-  const spec = alias ? findEvalModel(alias) : getDefaultEvalModel(mode);
-  if (!spec) {
-    throw new Error(`Unknown model: ${alias}`);
-  }
-
-  if (mode === "cli" && !spec.cli) {
-    throw new Error(`Model ${spec.id} is not supported for cli mode`);
-  }
-
-  if (mode !== "cli" && !spec.frontend) {
-    throw new Error(`Model ${spec.id} is not supported for ${mode} mode`);
-  }
-
-  return spec;
-}
-
-export function getEvalModelHelpText(): string {
-  return EVAL_MODELS.map((model) => {
-    const modes = [
-      ...(model.frontend ? ["flow", "script", "app"] : []),
-      ...(model.cli ? ["cli"] : []),
-    ];
-    return `  ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`;
-  }).join("\n");
-}
-
-export function formatRunModelLabel(mode: EvalMode, model: EvalModelSpec): string {
-  if (mode === "cli") {
-    return `${model.cli!.provider}:${model.cli!.model}`;
-  }
-  return `${model.frontend!.provider}:${model.frontend!.model}`;
-}
-
-export function getFrontendEvalModel(model: EvalModelSpec): FrontendEvalModelConfig {
-  if (!model.frontend) {
-    throw new Error(`Model ${model.id} does not support frontend evals`);
-  }
-  return model.frontend;
-}
-
-export function getCliEvalModel(model: EvalModelSpec): CliEvalModelConfig {
-  if (!model.cli) {
-    throw new Error(`Model ${model.id} does not support cli evals`);
-  }
-  return model.cli;
-}
-
-function getDefaultEvalModel(mode: EvalMode): EvalModelSpec {
-  return mode === "cli" ? EVAL_MODELS[0]! : EVAL_MODELS[0]!;
-}
-
-function findEvalModel(alias: string): EvalModelSpec | undefined {
-  const normalized = alias.trim().toLowerCase();
-  return EVAL_MODELS.find((model) =>
-    [model.id, ...model.aliases].some((candidate) => candidate.toLowerCase() === normalized)
-  );
-}

ai_evals/core/results.ts

@@ -1,216 +0,0 @@
-import { appendFile, mkdir, rm, writeFile } from "node:fs/promises";
-import path from "node:path";
-import { execFileSync } from "node:child_process";
-import { getAiEvalsRoot, getRepoRoot } from "./cases";
-import type {
-  BenchmarkArtifactFile,
-  BenchmarkCaseResult,
-  BenchmarkRunResult,
-  EvalMode,
-} from "./types";
-
-export async function writeRunResult(
-  result: BenchmarkRunResult,
-  outputPath?: string
-): Promise<string> {
-  const targetPath = resolveRunOutputPath(result.mode, outputPath);
-  await mkdir(path.dirname(targetPath), { recursive: true });
-  await writeFile(targetPath, JSON.stringify(toSerializableRunResult(result), null, 2) + "\n", "utf8");
-  return targetPath;
-}
-
-export async function appendHistoryRecord(
-  result: BenchmarkRunResult,
-  historyPath = resolveHistoryPath(result.mode)
-): Promise<string> {
-  await mkdir(path.dirname(historyPath), { recursive: true });
-  await appendFile(historyPath, JSON.stringify(toHistoryRecord(result)) + "\n", "utf8");
-  return historyPath;
-}
-
-export async function writeRunArtifacts(
-  result: BenchmarkRunResult,
-  outputPath?: string
-): Promise<string | null> {
-  const targetPath = resolveRunOutputPath(result.mode, outputPath);
-  const artifactRoot = defaultArtifactsRoot(targetPath);
-
-  await rm(artifactRoot, { recursive: true, force: true });
-
-  let wroteArtifacts = false;
-  for (const caseResult of result.cases) {
-    for (const attempt of caseResult.attempts) {
-      const artifactFiles = attempt.artifactFiles ?? [];
-      if (artifactFiles.length === 0) {
-        attempt.artifactsPath = null;
-        continue;
-      }
-
-      const attemptDir = path.join(artifactRoot, caseResult.id, `attempt-${attempt.attempt}`);
-      await writeArtifactFiles(attemptDir, artifactFiles);
-      attempt.artifactsPath = attemptDir;
-      wroteArtifacts = true;
-    }
-  }
-
-  result.artifactsPath = wroteArtifacts ? artifactRoot : null;
-  return result.artifactsPath ?? null;
-}
-
-export function buildRunResult(input: {
-  mode: EvalMode;
-  runs: number;
-  runModel: string | null;
-  judgeModel: string | null;
-  caseResults: BenchmarkCaseResult[];
-}): BenchmarkRunResult {
-  const attemptCount = input.caseResults.reduce((sum, entry) => sum + entry.attempts.length, 0);
-  const passedAttempts = input.caseResults.reduce(
-    (sum, entry) => sum + entry.attempts.filter((attempt) => attempt.passed).length,
-    0
-  );
-  const durationTotal = input.caseResults.reduce(
-    (sum, entry) => sum + entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
-    0
-  );
-
-  return {
-    version: 1,
-    mode: input.mode,
-    createdAt: new Date().toISOString(),
-    gitSha: getGitSha(),
-    runs: input.runs,
-    runModel: input.runModel,
-    judgeModel: input.judgeModel,
-    caseCount: input.caseResults.length,
-    attemptCount,
-    passedAttempts,
-    passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
-    averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount,
-    cases: input.caseResults,
-  };
-}
-
-export function formatRunSummary(result: BenchmarkRunResult): string {
-  const lines = [
-    `${result.mode} benchmark complete`,
-    `Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
-    `Average duration: ${Math.round(result.averageDurationMs)}ms`,
-  ];
-
-  const failures = collectFailures(result);
-  if (failures.length > 0) {
-    lines.push("Failures:");
-    for (const entry of failures.slice(0, 10)) {
-      lines.push(`- ${entry}`);
-    }
-  }
-
-  return lines.join("\n");
-}
-
-function collectFailures(result: BenchmarkRunResult): string[] {
-  const failures: string[] = [];
-
-  for (const caseResult of result.cases) {
-    for (const attempt of caseResult.attempts) {
-      if (attempt.passed) {
-        continue;
-      }
-      const failedChecks = attempt.checks.filter((check) => !check.passed).map((check) => check.name);
-      failures.push(
-        `${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}`
-      );
-    }
-  }
-
-  return failures;
-}
-
-function defaultFileName(mode: EvalMode): string {
-  return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
-}
-
-export function resolveRunOutputPath(mode: EvalMode, outputPath?: string): string {
-  return outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode));
-}
-
-export function resolveHistoryPath(mode: EvalMode): string {
-  return path.join(getAiEvalsRoot(), "history", `${mode}.jsonl`);
-}
-
-function defaultArtifactsRoot(resultPath: string): string {
-  return resultPath.endsWith(".json")
-    ? resultPath.slice(0, -".json".length)
-    : `${resultPath}.artifacts`;
-}
-
-async function writeArtifactFiles(
-  rootDir: string,
-  files: BenchmarkArtifactFile[]
-): Promise<void> {
-  for (const file of files) {
-    const relativePath = normalizeArtifactPath(file.path);
-    const targetPath = path.join(rootDir, relativePath);
-    await mkdir(path.dirname(targetPath), { recursive: true });
-    await writeFile(targetPath, file.content, "utf8");
-  }
-}
-
-function normalizeArtifactPath(filePath: string): string {
-  const normalized = filePath.replaceAll("\\", "/").replace(/^\/+/, "");
-  const parts = normalized.split("/").filter(Boolean);
-  if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) {
-    throw new Error(`Invalid artifact path: ${filePath}`);
-  }
-  return parts.join("/");
-}
-
-function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult {
-  return {
-    ...result,
-    cases: result.cases.map((caseResult) => ({
-      ...caseResult,
-      attempts: caseResult.attempts.map(({ artifactFiles, ...attempt }) => attempt),
-    })),
-  };
-}
-
-function toHistoryRecord(result: BenchmarkRunResult) {
-  return {
-    createdAt: result.createdAt,
-    gitSha: result.gitSha,
-    mode: result.mode,
-    runs: result.runs,
-    runModel: result.runModel,
-    judgeModel: result.judgeModel,
-    caseCount: result.caseCount,
-    attemptCount: result.attemptCount,
-    passedAttempts: result.passedAttempts,
-    passRate: result.passRate,
-    averageDurationMs: result.averageDurationMs,
-    failedCaseIds: Array.from(
-      new Set(
-        result.cases
-          .filter((caseResult) => caseResult.attempts.some((attempt) => !attempt.passed))
-          .map((caseResult) => caseResult.id)
-      )
-    ),
-  };
-}
-
-function getGitSha(): string | null {
-  try {
-    return execFileSync("git", ["rev-parse", "HEAD"], {
-      cwd: getRepoRoot(),
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-    }).trim();
-  } catch {
-    return null;
-  }
-}
-
-function formatPercent(value: number): string {
-  return `${(value * 100).toFixed(1)}%`;
-}

ai_evals/core/runSuite.ts

@@ -1,255 +0,0 @@
-import { judgeOutput, DEFAULT_JUDGE_MODEL } from "./judge";
-import type {
-  BenchmarkAttemptResult,
-  BenchmarkCaseResult,
-  BenchmarkCheck,
-  EvalCase,
-  FrontendBenchmarkProgressEvent,
-  ModeRunner,
-} from "./types";
-
-export async function runSuite<TInitial, TExpected, TActual>(input: {
-  modeRunner: ModeRunner<TInitial, TExpected, TActual>;
-  cases: EvalCase[];
-  runs: number;
-  runModel: string | null;
-  judgeModel?: string | null;
-  concurrency?: number;
-  verbose?: boolean;
-  onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
-}): Promise<BenchmarkCaseResult[]> {
-  const judgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;
-  const concurrency = Math.max(1, input.concurrency ?? input.modeRunner.concurrency);
-  const results = new Array<BenchmarkCaseResult>(input.cases.length);
-  let cursor = 0;
-
-  if (input.modeRunner.mode !== "cli") {
-    input.onProgress?.({
-      type: "run-start",
-      surface: input.modeRunner.mode,
-      totalCases: input.cases.length,
-      runs: input.runs,
-      concurrency,
-    });
-  }
-
-  async function worker(): Promise<void> {
-    while (true) {
-      const caseIndex = cursor++;
-      if (caseIndex >= input.cases.length) {
-        return;
-      }
-      const evalCase = input.cases[caseIndex];
-      results[caseIndex] = {
-        id: evalCase.id,
-        prompt: evalCase.prompt,
-        initialPath: evalCase.initialPath,
-        expectedPath: evalCase.expectedPath,
-        attempts: await runCaseAttempts({
-          caseIndex,
-          evalCase,
-          runs: input.runs,
-          judgeModel,
-          judgeThreshold: input.modeRunner.judgeThreshold ?? 80,
-          modeRunner: input.modeRunner,
-          totalCases: input.cases.length,
-          verbose: input.verbose ?? false,
-          onProgress: input.onProgress,
-        }),
-      };
-    }
-  }
-
-  await Promise.all(
-    Array.from({ length: Math.min(concurrency, input.cases.length) }, () => worker())
-  );
-
-  return results;
-}
-
-async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
-  caseIndex: number;
-  evalCase: EvalCase;
-  runs: number;
-  judgeModel: string;
-  judgeThreshold: number;
-  modeRunner: ModeRunner<TInitial, TExpected, TActual>;
-  totalCases: number;
-  verbose: boolean;
-  onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
-}): Promise<BenchmarkAttemptResult[]> {
-  const attempts: BenchmarkAttemptResult[] = [];
-
-  for (let attempt = 1; attempt <= input.runs; attempt += 1) {
-    input.onProgress?.({
-      type: "attempt-start",
-      surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-      caseId: input.evalCase.id,
-      caseNumber: input.caseIndex + 1,
-      totalCases: input.totalCases,
-      attempt,
-      runs: input.runs,
-    });
-
-    const startedAt = Date.now();
-    const initial = await input.modeRunner.loadInitial(input.evalCase.initialPath);
-    const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath);
-
-    try {
-      const run = await input.modeRunner.run(input.evalCase.prompt, initial, {
-        caseId: input.evalCase.id,
-        caseNumber: input.caseIndex + 1,
-        totalCases: input.totalCases,
-        attempt,
-        runs: input.runs,
-        verbose: input.verbose,
-        onAssistantMessageStart: input.verbose
-          ? () =>
-              input.onProgress?.({
-                type: "assistant-message-start",
-                surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-                caseId: input.evalCase.id,
-                caseNumber: input.caseIndex + 1,
-                totalCases: input.totalCases,
-                attempt,
-                runs: input.runs,
-              })
-          : undefined,
-        onAssistantChunk: input.verbose
-          ? (chunk: string) =>
-              input.onProgress?.({
-                type: "assistant-chunk",
-                surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-                caseId: input.evalCase.id,
-                caseNumber: input.caseIndex + 1,
-                totalCases: input.totalCases,
-                attempt,
-                runs: input.runs,
-                chunk,
-              })
-          : undefined,
-        onAssistantMessageEnd: input.verbose
-          ? () =>
-              input.onProgress?.({
-                type: "assistant-message-end",
-                surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-                caseId: input.evalCase.id,
-                caseNumber: input.caseIndex + 1,
-                totalCases: input.totalCases,
-                attempt,
-                runs: input.runs,
-              })
-          : undefined,
-      });
-      const checks: BenchmarkCheck[] = [
-        buildCheck("run succeeded", run.success, run.error),
-        ...input.modeRunner.validate({
-          evalCase: input.evalCase,
-          prompt: input.evalCase.prompt,
-          initial,
-          expected,
-          actual: run.actual,
-          run,
-        }),
-      ];
-
-      let judgeScore: number | null = null;
-      let judgeSummary: string | null = null;
-
-      if (run.success) {
-        const judge = await judgeOutput({
-          mode: input.modeRunner.mode,
-          prompt: input.evalCase.prompt,
-          checklist: input.evalCase.judgeChecklist,
-          initial,
-          expected,
-          actual: run.actual,
-          model: input.judgeModel,
-        });
-
-        judgeScore = judge.success ? judge.score : null;
-        judgeSummary = judge.summary;
-        checks.push(buildCheck("judge succeeded", judge.success, judge.error));
-        checks.push(
-          buildCheck(
-            `judge score >= ${input.judgeThreshold}`,
-            (judgeScore ?? 0) >= input.judgeThreshold,
-            judge.success ? `score=${judgeScore}` : judge.error
-          )
-        );
-      }
-
-      const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? [];
-      const attemptResult: BenchmarkAttemptResult = {
-        attempt,
-        passed: checks.every((check) => check.passed),
-        durationMs: Date.now() - startedAt,
-        assistantMessageCount: run.assistantMessageCount,
-        toolCallCount: run.toolCallCount,
-        toolsUsed: uniqueStrings(run.toolsUsed),
-        skillsInvoked: uniqueStrings(run.skillsInvoked),
-        checks,
-        judgeScore,
-        judgeSummary,
-        error: run.error ?? null,
-        artifactsPath: null,
-        artifactFiles,
-      };
-
-      input.onProgress?.({
-        type: "attempt-finish",
-        surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-        caseId: input.evalCase.id,
-        caseNumber: input.caseIndex + 1,
-        totalCases: input.totalCases,
-        attempt,
-        runs: input.runs,
-        passed: attemptResult.passed,
-        durationMs: attemptResult.durationMs,
-        judgeScore: attemptResult.judgeScore,
-        error: attemptResult.error,
-      });
-
-      attempts.push(attemptResult);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      const failedAttempt: BenchmarkAttemptResult = {
-        attempt,
-        passed: false,
-        durationMs: Date.now() - startedAt,
-        assistantMessageCount: 0,
-        toolCallCount: 0,
-        toolsUsed: [],
-        skillsInvoked: [],
-        checks: [buildCheck("run crashed", false, message)],
-        judgeScore: null,
-        judgeSummary: null,
-        error: message,
-      };
-      input.onProgress?.({
-        type: "attempt-finish",
-        surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
-        caseId: input.evalCase.id,
-        caseNumber: input.caseIndex + 1,
-        totalCases: input.totalCases,
-        attempt,
-        runs: input.runs,
-        passed: false,
-        durationMs: failedAttempt.durationMs,
-        judgeScore: null,
-        error: message,
-      });
-      attempts.push(failedAttempt);
-    }
-  }
-
-  return attempts;
-}
-
-function buildCheck(name: string, passed: boolean, details?: string): BenchmarkCheck {
-  return details ? { name, passed, details } : { name, passed };
-}
-
-function uniqueStrings(values: string[]): string[] {
-  return [...new Set(values)];
-}

ai_evals/core/types.ts

@@ -1,188 +0,0 @@
-export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
-
-export type EvalMode = (typeof EVAL_MODES)[number];
-
-export interface FlowValidationSpec {
-  schemaRequiredPaths?: string[];
-  schemaAnyOf?: Array<{
-    requiredPaths: string[];
-  }>;
-  resolveResultsRefs?: boolean;
-  requireSpecialModules?: Array<"preprocessor_module" | "failure_module">;
-  requireSuspendSteps?: Array<{
-    id: string;
-    requiredEvents?: number;
-    resumeRequiredStringFieldAnyOf?: string[];
-  }>;
-}
-
-export interface EvalCase {
-  id: string;
-  prompt: string;
-  initialPath?: string;
-  expectedPath?: string;
-  validate?: FlowValidationSpec;
-  judgeChecklist?: string[];
-}
-
-export interface BenchmarkCheck {
-  name: string;
-  passed: boolean;
-  details?: string;
-}
-
-export interface JudgeResult {
-  success: boolean;
-  score: number;
-  summary: string;
-  error?: string;
-}
-
-export interface BenchmarkArtifactFile {
-  path: string;
-  content: string;
-}
-
-export interface ModeRunOutput<TActual> {
-  success: boolean;
-  actual: TActual;
-  error?: string;
-  assistantMessageCount: number;
-  toolCallCount: number;
-  toolsUsed: string[];
-  skillsInvoked: string[];
-}
-
-export interface ModeRunContext {
-  caseId: string;
-  caseNumber: number;
-  totalCases: number;
-  attempt: number;
-  runs: number;
-  verbose: boolean;
-  onAssistantMessageStart?: () => void;
-  onAssistantChunk?: (chunk: string) => void;
-  onAssistantMessageEnd?: () => void;
-}
-
-export interface ModeRunner<TInitial, TExpected, TActual> {
-  mode: EvalMode;
-  concurrency: number;
-  judgeThreshold?: number;
-  loadInitial(path?: string): Promise<TInitial | undefined>;
-  loadExpected(path?: string): Promise<TExpected | undefined>;
-  run(
-    prompt: string,
-    initial: TInitial | undefined,
-    context: ModeRunContext
-  ): Promise<ModeRunOutput<TActual>>;
-  validate(input: {
-    evalCase: EvalCase;
-    prompt: string;
-    initial: TInitial | undefined;
-    expected: TExpected | undefined;
-    actual: TActual;
-    run: ModeRunOutput<TActual>;
-  }): BenchmarkCheck[];
-  buildArtifacts?(actual: TActual): BenchmarkArtifactFile[];
-}
-
-export interface BenchmarkAttemptResult {
-  attempt: number;
-  passed: boolean;
-  durationMs: number;
-  assistantMessageCount: number;
-  toolCallCount: number;
-  toolsUsed: string[];
-  skillsInvoked: string[];
-  checks: BenchmarkCheck[];
-  judgeScore: number | null;
-  judgeSummary: string | null;
-  error: string | null;
-  artifactsPath?: string | null;
-  artifactFiles?: BenchmarkArtifactFile[];
-}
-
-export interface BenchmarkCaseResult {
-  id: string;
-  prompt: string;
-  initialPath?: string;
-  expectedPath?: string;
-  attempts: BenchmarkAttemptResult[];
-}
-
-export interface BenchmarkRunResult {
-  version: 1;
-  mode: EvalMode;
-  createdAt: string;
-  gitSha: string | null;
-  runs: number;
-  runModel: string | null;
-  judgeModel: string | null;
-  caseCount: number;
-  attemptCount: number;
-  passedAttempts: number;
-  passRate: number;
-  averageDurationMs: number;
-  artifactsPath?: string | null;
-  cases: BenchmarkCaseResult[];
-}
-
-export type FrontendBenchmarkProgressEvent =
-  | {
-      type: "run-start";
-      surface: Exclude<EvalMode, "cli">;
-      totalCases: number;
-      runs: number;
-      concurrency: number;
-    }
-  | {
-      type: "attempt-start";
-      surface: Exclude<EvalMode, "cli">;
-      caseId: string;
-      caseNumber: number;
-      totalCases: number;
-      attempt: number;
-      runs: number;
-    }
-  | {
-      type: "attempt-finish";
-      surface: Exclude<EvalMode, "cli">;
-      caseId: string;
-      caseNumber: number;
-      totalCases: number;
-      attempt: number;
-      runs: number;
-      passed: boolean;
-      durationMs: number;
-      judgeScore: number | null;
-      error: string | null;
-    }
-  | {
-      type: "assistant-message-start";
-      surface: Exclude<EvalMode, "cli">;
-      caseId: string;
-      caseNumber: number;
-      totalCases: number;
-      attempt: number;
-      runs: number;
-    }
-  | {
-      type: "assistant-chunk";
-      surface: Exclude<EvalMode, "cli">;
-      caseId: string;
-      caseNumber: number;
-      totalCases: number;
-      attempt: number;
-      runs: number;
-      chunk: string;
-    }
-  | {
-      type: "assistant-message-end";
-      surface: Exclude<EvalMode, "cli">;
-      caseId: string;
-      caseNumber: number;
-      totalCases: number;
-      attempt: number;
-      runs: number;
-    };

ai_evals/core/validators.ts

@@ -1,712 +0,0 @@
-import ts from "typescript";
-import type { BenchmarkCheck, FlowValidationSpec } from "./types";
-
-export interface ScriptState {
-  path: string;
-  lang: string;
-  args?: Record<string, unknown>;
-  code: string;
-}
-
-export interface FlowState {
-  value?: {
-    modules?: Array<Record<string, unknown>>;
-  };
-  schema?: Record<string, unknown>;
-}
-
-export interface AppFilesState {
-  frontend: Record<string, string>;
-  backend: Record<string, AppRunnableState>;
-}
-
-export interface AppRunnableState {
-  type?: string;
-  name?: string;
-  path?: string;
-  inlineScript?: {
-    language?: string;
-    content?: string;
-  };
-}
-
-const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
-const CONTROL_FLOW_MODULE_TYPES = new Set(["branchone", "branchall", "forloopflow", "whileloopflow"]);
-
-export function validateScriptState(input: {
-  actual: ScriptState;
-  initial?: ScriptState;
-  expected?: ScriptState;
-}): BenchmarkCheck[] {
-  const checks: BenchmarkCheck[] = [
-    check("script exports entrypoint", hasSupportedEntrypoint(input.actual.code)),
-    check("script has no syntax errors", getScriptSyntaxErrors(input.actual.code, input.actual.lang).length === 0),
-  ];
-
-  if (input.expected) {
-    checks.push(
-      check(
-        "script path matches expected",
-        input.actual.path === input.expected.path,
-        `expected ${input.expected.path}, got ${input.actual.path}`
-      )
-    );
-    checks.push(
-      check(
-        "script language matches expected",
-        input.actual.lang === input.expected.lang,
-        `expected ${input.expected.lang}, got ${input.actual.lang}`
-      )
-    );
-    checks.push(
-      check(
-        "script code matches expected",
-        normalizeText(input.actual.code) === normalizeText(input.expected.code)
-      )
-    );
-  }
-
-  if (input.initial) {
-    checks.push(
-      check(
-        "script differs from initial",
-        normalizeText(input.actual.code) !== normalizeText(input.initial.code)
-      )
-    );
-  }
-
-  return checks;
-}
-
-export function validateFlowState(input: {
-  actual: FlowState;
-  initial?: FlowState;
-  validate?: FlowValidationSpec;
-}): BenchmarkCheck[] {
-  const actualModules = getFlowModules(input.actual);
-  const placeholderModuleIds = getInlineScriptPlaceholderModuleIds(input.actual);
-  const checks: BenchmarkCheck[] = [
-    check("flow has modules", actualModules.length > 0),
-    check(
-      "flow has no inline placeholder code",
-      placeholderModuleIds.length === 0,
-      placeholderModuleIds.length > 0
-        ? `placeholder content in: ${placeholderModuleIds.join(", ")}`
-        : undefined
-    ),
-  ];
-
-  if (input.initial) {
-    checks.push(
-      check(
-        "flow differs from initial",
-        normalizeJson(input.actual) !== normalizeJson(input.initial)
-      )
-    );
-  }
-
-  if (input.validate) {
-    checks.push(...validateFlowRequirements(input.actual, input.validate));
-  }
-
-  return checks;
-}
-
-export function validateAppState(input: {
-  actual: AppFilesState;
-  initial?: AppFilesState;
-  expected?: AppFilesState;
-}): BenchmarkCheck[] {
-  const checks: BenchmarkCheck[] = [];
-  const frontendEntries = Object.entries(input.actual.frontend ?? {});
-  const backendEntries = Object.entries(input.actual.backend ?? {});
-
-  checks.push(check("app has frontend entrypoint", Boolean(input.actual.frontend["/index.tsx"])));
-  checks.push(check("app has non-empty frontend files", frontendEntries.some(([, content]) => content.trim().length > 0)));
-  checks.push(
-    check(
-      "backend inline scripts have entrypoints",
-      backendEntries.every(([, runnable]) => {
-        if (runnable.type !== "inline") {
-          return true;
-        }
-        return hasSupportedEntrypoint(runnable.inlineScript?.content ?? "");
-      })
-    )
-  );
-
-  if (input.initial) {
-    checks.push(check("app differs from initial", !appStatesEqual(input.actual, input.initial)));
-  }
-
-  if (input.expected) {
-    for (const [filePath, content] of Object.entries(input.expected.frontend)) {
-      checks.push(
-        check(
-          `frontend includes ${filePath}`,
-          normalizeText(input.actual.frontend[filePath] ?? "") === normalizeText(content)
-        )
-      );
-    }
-    for (const [runnableName, runnable] of Object.entries(input.expected.backend)) {
-      const actualRunnable = input.actual.backend[runnableName];
-      checks.push(check(`backend includes ${runnableName}`, Boolean(actualRunnable)));
-      if (actualRunnable && runnable.inlineScript?.content) {
-        checks.push(
-          check(
-            `${runnableName} code matches expected`,
-            normalizeText(actualRunnable.inlineScript?.content ?? "") ===
-              normalizeText(runnable.inlineScript.content)
-          )
-        );
-      }
-    }
-  }
-
-  return checks;
-}
-
-export function validateCliWorkspace(input: {
-  actualFiles: Record<string, string>;
-  expectedFiles?: Record<string, string>;
-  initialFiles?: Record<string, string>;
-}): BenchmarkCheck[] {
-  const checks: BenchmarkCheck[] = [];
-
-  if (input.expectedFiles) {
-    for (const [filePath, expectedContent] of Object.entries(input.expectedFiles)) {
-      const actualContent = input.actualFiles[filePath];
-      checks.push(check(`creates ${filePath}`, actualContent !== undefined));
-      if (actualContent !== undefined) {
-        checks.push(
-          check(
-            `${filePath} contains expected content`,
-            actualContent.includes(expectedContent.trim())
-          )
-        );
-      }
-    }
-  }
-
-  if (input.initialFiles) {
-    checks.push(check("workspace differs from initial", !fileMapsEqual(input.actualFiles, input.initialFiles)));
-  }
-
-  return checks;
-}
-
-function check(name: string, passed: boolean, details?: string): BenchmarkCheck {
-  return !passed && details ? { name, passed, details } : { name, passed };
-}
-
-function normalizeText(value: string): string {
-  return value.replace(/\r\n/g, "\n").trim();
-}
-
-function normalizeJson(value: unknown): string {
-  return JSON.stringify(value);
-}
-
-function hasSupportedEntrypoint(code: string): boolean {
-  return (
-    /export\s+(async\s+)?function\s+main\s*\(/.test(code) ||
-    /export\s+default\s+(async\s+)?function\s*\(/.test(code)
-  );
-}
-
-function getScriptSyntaxErrors(code: string, lang: string): string[] {
-  if (!TS_LIKE_LANGUAGES.has(lang)) {
-    return [];
-  }
-
-  const result = ts.transpileModule(code, {
-    compilerOptions: {
-      target: ts.ScriptTarget.ES2022,
-      module: ts.ModuleKind.ESNext,
-    },
-    reportDiagnostics: true,
-    fileName: "eval.ts",
-  });
-
-  return (result.diagnostics ?? []).map((diagnostic) =>
-    ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
-  );
-}
-
-function getFlowModules(flow: FlowState): Array<Record<string, unknown>> {
-  return Array.isArray(flow.value?.modules) ? flow.value.modules : [];
-}
-
-function validateFlowRequirements(
-  flow: FlowState,
-  validate: FlowValidationSpec
-): BenchmarkCheck[] {
-  const checks: BenchmarkCheck[] = [];
-
-  for (const requiredPath of validate.schemaRequiredPaths ?? []) {
-    checks.push(
-      check(
-        `schema includes ${requiredPath}`,
-        hasSchemaPath(flow.schema, requiredPath),
-        `missing schema path ${requiredPath}`
-      )
-    );
-  }
-
-  if (validate.schemaAnyOf && validate.schemaAnyOf.length > 0) {
-    const matchingVariant = validate.schemaAnyOf.find((variant) =>
-      variant.requiredPaths.every((requiredPath) => hasSchemaPath(flow.schema, requiredPath))
-    );
-
-    checks.push(
-      check(
-        "schema matches one accepted input shape",
-        Boolean(matchingVariant),
-        matchingVariant
-          ? undefined
-          : `expected one of: ${validate.schemaAnyOf
-              .map((variant) => `[${variant.requiredPaths.join(", ")}]`)
-              .join(" or ")}`
-      )
-    );
-  }
-
-  if (validate.resolveResultsRefs) {
-    const unresolved = collectUnresolvedResultsRefs(flow);
-    checks.push(
-      check(
-        "results references resolve",
-        unresolved.length === 0,
-        unresolved.length > 0 ? unresolved.join("; ") : undefined
-      )
-    );
-  }
-
-  for (const specialModule of validate.requireSpecialModules ?? []) {
-    checks.push(
-      check(
-        `${specialModule} exists`,
-        Boolean(getSpecialFlowModule(flow, specialModule))
-      )
-    );
-  }
-
-  for (const suspendStep of validate.requireSuspendSteps ?? []) {
-    const module = findFlowModuleById(flow, suspendStep.id);
-    checks.push(check(`${suspendStep.id} step exists`, Boolean(module)));
-    if (!module) {
-      continue;
-    }
-
-    checks.push(check(`${suspendStep.id} includes suspend config`, hasSuspendConfig(module)));
-    if (!hasSuspendConfig(module)) {
-      continue;
-    }
-
-    if (suspendStep.requiredEvents !== undefined) {
-      checks.push(
-        check(
-          `${suspendStep.id} requires ${suspendStep.requiredEvents} approval event${suspendStep.requiredEvents === 1 ? "" : "s"}`,
-          getSuspendRequiredEvents(module) === suspendStep.requiredEvents,
-          `expected ${suspendStep.requiredEvents}, got ${getSuspendRequiredEvents(module) ?? "(missing)"}`
-        )
-      );
-    }
-
-    if (
-      suspendStep.resumeRequiredStringFieldAnyOf &&
-      suspendStep.resumeRequiredStringFieldAnyOf.length > 0
-    ) {
-      const requiredFields = getSuspendResumeRequiredStringFields(module);
-      checks.push(
-        check(
-          `${suspendStep.id} resume form requires one accepted comment field`,
-          suspendStep.resumeRequiredStringFieldAnyOf.some((field) =>
-            requiredFields.includes(field)
-          ),
-          `required one of [${suspendStep.resumeRequiredStringFieldAnyOf.join(", ")}], got [${requiredFields.join(", ")}]`
-        )
-      );
-    }
-  }
-
-  return checks;
-}
-
-function hasSchemaPath(schema: Record<string, unknown> | undefined, dottedPath: string): boolean {
-  if (!schema || typeof schema !== "object") {
-    return false;
-  }
-
-  const segments = dottedPath.split(".").filter(Boolean);
-  if (segments.length === 0) {
-    return false;
-  }
-
-  let current: Record<string, unknown> | undefined = schema;
-  for (const segment of segments) {
-    const properties = current?.properties;
-    if (!properties || typeof properties !== "object") {
-      return false;
-    }
-
-    const next = (properties as Record<string, unknown>)[segment];
-    if (!next || typeof next !== "object") {
-      return false;
-    }
-    current = next as Record<string, unknown>;
-  }
-
-  return true;
-}
-
-function collectUnresolvedResultsRefs(flow: FlowState): string[] {
-  const unresolved = new Set<string>();
-  validateModuleSequence(getFlowModules(flow), new Map<string, Record<string, unknown>>(), unresolved);
-  return [...unresolved];
-}
-
-function validateModuleSequence(
-  modules: Array<Record<string, unknown>>,
-  parentVisibleModules: Map<string, Record<string, unknown>>,
-  unresolved: Set<string>
-): void {
-  const visibleModules = new Map(parentVisibleModules);
-
-  for (const module of modules) {
-    validateResultsRefsInRecord(module, visibleModules, unresolved);
-    validateNestedModuleResultsRefs(module, visibleModules, unresolved);
-
-    if (typeof module.id === "string" && module.id.length > 0) {
-      visibleModules.set(module.id, module);
-    }
-  }
-}
-
-function validateNestedModuleResultsRefs(
-  module: Record<string, unknown>,
-  visibleModules: Map<string, Record<string, unknown>>,
-  unresolved: Set<string>
-): void {
-  const value = isObjectRecord(module.value) ? module.value : null;
-  if (!value) {
-    return;
-  }
-
-  const nestedSequences: Array<Array<Record<string, unknown>>> = [];
-
-  if (Array.isArray(value.modules)) {
-    nestedSequences.push(asModuleArray(value.modules));
-  }
-
-  if (Array.isArray(value.default)) {
-    nestedSequences.push(asModuleArray(value.default));
-  }
-
-  if (Array.isArray(value.branches)) {
-    for (const branch of value.branches) {
-      if (!isObjectRecord(branch)) {
-        continue;
-      }
-      if (typeof branch.expr === "string") {
-        validateResultsRefsInExpression(
-          branch.expr,
-          `branch ${module.id ?? "(unnamed)"}`,
-          visibleModules,
-          unresolved
-        );
-      }
-      if (Array.isArray(branch.modules)) {
-        nestedSequences.push(asModuleArray(branch.modules));
-      }
-    }
-  }
-
-  for (const sequence of nestedSequences) {
-    validateModuleSequence(sequence, visibleModules, unresolved);
-  }
-}
-
-function validateResultsRefsInRecord(
-  value: unknown,
-  visibleModules: Map<string, Record<string, unknown>>,
-  unresolved: Set<string>,
-  context = "expression"
-): void {
-  if (typeof value === "string") {
-    validateResultsRefsInExpression(value, context, visibleModules, unresolved);
-    return;
-  }
-
-  if (Array.isArray(value)) {
-    for (const entry of value) {
-      validateResultsRefsInRecord(entry, visibleModules, unresolved, context);
-    }
-    return;
-  }
-
-  if (!isObjectRecord(value)) {
-    return;
-  }
-
-  for (const [key, entry] of Object.entries(value)) {
-    if (key === "content" || key === "modules" || key === "branches" || key === "default") {
-      continue;
-    }
-    validateResultsRefsInRecord(entry, visibleModules, unresolved, key);
-  }
-}
-
-function validateResultsRefsInExpression(
-  expression: string,
-  context: string,
-  visibleModules: Map<string, Record<string, unknown>>,
-  unresolved: Set<string>
-): void {
-  for (const ref of extractResultsRefs(expression)) {
-    const module = visibleModules.get(ref.root);
-    if (!module) {
-      unresolved.add(`${context} references missing results.${ref.root}`);
-      continue;
-    }
-    validateNestedResultsRefPath(ref.root, ref.path, module, context, unresolved);
-  }
-}
-
-function extractResultsRefs(
-  expression: string
-): Array<{ root: string; path: string[] }> {
-  const matches = expression.matchAll(/\bresults\.([A-Za-z0-9_-]+)((?:\.[A-Za-z0-9_-]+)*)/g);
-  const refs = new Map<string, { root: string; path: string[] }>();
-
-  for (const match of matches) {
-    const root = match[1];
-    const path = match[2]
-      .split(".")
-      .filter(Boolean);
-    const key = `${root}:${path.join(".")}`;
-    refs.set(key, { root, path });
-  }
-
-  return [...refs.values()];
-}
-
-function validateNestedResultsRefPath(
-  rootId: string,
-  path: string[],
-  module: Record<string, unknown>,
-  context: string,
-  unresolved: Set<string>
-): void {
-  if (path.length === 0) {
-    return;
-  }
-
-  const moduleType = getModuleType(module);
-  if (!moduleType || !CONTROL_FLOW_MODULE_TYPES.has(moduleType)) {
-    return;
-  }
-
-  const nestedIds = new Set(getImmediateNestedModuleIds(module));
-  const [firstSegment] = path;
-  if (nestedIds.has(firstSegment)) {
-    unresolved.add(
-      `${context} references nested results.${rootId}.${firstSegment} inside ${moduleType} ${rootId}`
-    );
-  }
-}
-
-function getAllFlowModules(flow: FlowState): Array<Record<string, unknown>> {
-  const modules: Array<Record<string, unknown>> = [];
-  const specialModules = ["preprocessor_module", "failure_module"] as const;
-
-  for (const key of specialModules) {
-    const specialModule = getSpecialFlowModule(flow, key);
-    if (specialModule) {
-      modules.push(specialModule);
-      modules.push(...collectNestedModules(specialModule));
-    }
-  }
-
-  for (const module of getFlowModules(flow)) {
-    modules.push(module);
-    modules.push(...collectNestedModules(module));
-  }
-
-  return modules;
-}
-
-function collectNestedModules(module: Record<string, unknown>): Array<Record<string, unknown>> {
-  const nested: Array<Record<string, unknown>> = [];
-  const value = isObjectRecord(module.value) ? module.value : null;
-  if (!value) {
-    return nested;
-  }
-
-  if (Array.isArray(value.modules)) {
-    for (const child of asModuleArray(value.modules)) {
-      nested.push(child, ...collectNestedModules(child));
-    }
-  }
-
-  if (Array.isArray(value.default)) {
-    for (const child of asModuleArray(value.default)) {
-      nested.push(child, ...collectNestedModules(child));
-    }
-  }
-
-  if (Array.isArray(value.branches)) {
-    for (const branch of value.branches) {
-      if (!isObjectRecord(branch) || !Array.isArray(branch.modules)) {
-        continue;
-      }
-      for (const child of asModuleArray(branch.modules)) {
-        nested.push(child, ...collectNestedModules(child));
-      }
-    }
-  }
-
-  return nested;
-}
-
-function findFlowModuleById(flow: FlowState, id: string): Record<string, unknown> | null {
-  for (const module of getAllFlowModules(flow)) {
-    if (module.id === id) {
-      return module;
-    }
-  }
-  return null;
-}
-
-function getInlineScriptPlaceholderModuleIds(flow: FlowState): string[] {
-  return getAllFlowModules(flow).flatMap((module) => {
-    const code = getModuleCode(module)?.trim();
-    if (!code || !/^inline_script\.[A-Za-z0-9_-]+$/.test(code)) {
-      return [];
-    }
-
-    if (typeof module.id === "string" && module.id.length > 0) {
-      return [module.id];
-    }
-
-    return ["(unnamed)"];
-  });
-}
-
-function getImmediateNestedModuleIds(module: Record<string, unknown>): string[] {
-  const ids: string[] = [];
-  const value = isObjectRecord(module.value) ? module.value : null;
-  if (!value) {
-    return ids;
-  }
-
-  if (Array.isArray(value.modules)) {
-    ids.push(...asModuleArray(value.modules).flatMap((child) => (typeof child.id === "string" ? [child.id] : [])));
-  }
-
-  if (Array.isArray(value.default)) {
-    ids.push(...asModuleArray(value.default).flatMap((child) => (typeof child.id === "string" ? [child.id] : [])));
-  }
-
-  if (Array.isArray(value.branches)) {
-    for (const branch of value.branches) {
-      if (!isObjectRecord(branch) || !Array.isArray(branch.modules)) {
-        continue;
-      }
-      ids.push(
-        ...asModuleArray(branch.modules).flatMap((child) => (typeof child.id === "string" ? [child.id] : []))
-      );
-    }
-  }
-
-  return ids;
-}
-
-function getModuleCode(module: Record<string, unknown>): string | null {
-  const value = isObjectRecord(module.value) ? module.value : null;
-  return typeof value?.content === "string" ? value.content : null;
-}
-
-function asModuleArray(value: unknown[]): Array<Record<string, unknown>> {
-  return value.filter(isObjectRecord);
-}
-
-function isObjectRecord(value: unknown): value is Record<string, any> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function getSpecialFlowModule(
-  flow: FlowState,
-  key: "preprocessor_module" | "failure_module"
-): Record<string, unknown> | null {
-  if (!flow.value || typeof flow.value !== "object") {
-    return null;
-  }
-  const module = (flow.value as Record<string, unknown>)[key];
-  return module && typeof module === "object" ? (module as Record<string, unknown>) : null;
-}
-
-function getModuleType(module: Record<string, unknown>): string | null {
-  const value = module.value;
-  if (!value || typeof value !== "object") {
-    return null;
-  }
-  return typeof (value as Record<string, unknown>).type === "string"
-    ? ((value as Record<string, string>).type)
-    : null;
-}
-
-function hasSuspendConfig(module: Record<string, unknown>): boolean {
-  return typeof module.suspend === "object" && module.suspend !== null;
-}
-
-function getSuspendRequiredEvents(module: Record<string, unknown>): number | null {
-  const suspend = isObjectRecord(module.suspend) ? module.suspend : null;
-  return typeof suspend?.required_events === "number" ? suspend.required_events : null;
-}
-
-function getSuspendResumeRequiredStringFields(module: Record<string, unknown>): string[] {
-  const suspend = isObjectRecord(module.suspend) ? module.suspend : null;
-  const resumeForm = isObjectRecord(suspend?.resume_form) ? suspend.resume_form : null;
-  const schema = isObjectRecord(resumeForm?.schema) ? resumeForm.schema : null;
-  const required = Array.isArray(schema?.required) ? schema.required : [];
-  const properties = isObjectRecord(schema?.properties) ? schema.properties : null;
-  if (!properties) {
-    return [];
-  }
-
-  return required.flatMap((field) => {
-    if (typeof field !== "string") {
-      return [];
-    }
-    const property = properties[field];
-    if (!isObjectRecord(property) || property.type !== "string") {
-      return [];
-    }
-    return [field];
-  });
-}
-
-function appStatesEqual(left: AppFilesState, right: AppFilesState): boolean {
-  return fileMapsEqual(left.frontend, right.frontend) && fileMapsEqual(stringifyBackend(left.backend), stringifyBackend(right.backend));
-}
-
-function stringifyBackend(backend: Record<string, AppRunnableState>): Record<string, string> {
-  const result: Record<string, string> = {};
-  for (const [key, value] of Object.entries(backend)) {
-    result[key] = JSON.stringify(value);
-  }
-  return result;
-}
-
-function fileMapsEqual(left: Record<string, string>, right: Record<string, string>): boolean {
-  const leftEntries = Object.entries(left).sort(([a], [b]) => a.localeCompare(b));
-  const rightEntries = Object.entries(right).sort(([a], [b]) => a.localeCompare(b));
-  if (leftEntries.length !== rightEntries.length) {
-    return false;
-  }
-  return leftEntries.every(([key, value], index) => {
-    const [otherKey, otherValue] = rightEntries[index];
-    return key === otherKey && normalizeText(value) === normalizeText(otherValue);
-  });
-}

ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/flow.yaml

@@ -1,6 +0,0 @@
-value:
-  modules:
-Add File: /home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/hello.ts
-export async function main(name: string) {
-  return { greeting: `Hello, ${name}!` };
-}

ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/hello.ts

@@ -1,3 +0,0 @@
-export async function main(name: string) {
-  return { greeting: `Hello, ${name}!` };
-}

ai_evals/fixtures/cli/expected/bun-hello-script/f/evals/hello.ts

@@ -1,3 +0,0 @@
-export async function main(name: string) {
-  return { greeting: `Hello, ${name}!` };
-}

ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json

@@ -1,31 +0,0 @@
-{
-  "summary": "",
-  "value": {
-    "modules": [
-      {
-        "id": "sum_numbers",
-        "value": {
-          "type": "rawscript",
-          "language": "bun",
-          "content": "export async function main(a: number, b: number) {\n  return a + b;\n}",
-          "input_transforms": {
-            "a": {
-              "type": "javascript",
-              "expr": "flow_input.a"
-            },
-            "b": {
-              "type": "javascript",
-              "expr": "flow_input.b"
-            }
-          }
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "properties": {},
-    "required": [],
-    "type": "object"
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test10_while_loop_counter.json

@@ -1,30 +0,0 @@
-{
-  "value": {
-    "modules": [
-      {
-        "id": "count_until_target",
-        "value": {
-          "type": "whileloopflow"
-        }
-      },
-      {
-        "id": "return_final_count",
-        "value": {
-          "type": "rawscript"
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "target": {
-        "type": "number"
-      }
-    },
-    "required": [
-      "target"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test11_preprocessor_failure.json

@@ -1,36 +0,0 @@
-{
-  "value": {
-    "preprocessor_module": {
-      "id": "preprocessor",
-      "value": {
-        "type": "rawscript"
-      }
-    },
-    "failure_module": {
-      "id": "failure",
-      "value": {
-        "type": "rawscript"
-      }
-    },
-    "modules": [
-      {
-        "id": "process_event",
-        "value": {
-          "type": "rawscript"
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "payload": {
-        "type": "string"
-      }
-    },
-    "required": [
-      "payload"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test12_approval_step.json

@@ -1,44 +0,0 @@
-{
-  "value": {
-    "modules": [
-      {
-        "id": "request_approval",
-        "suspend": {
-          "required_events": 1,
-          "resume_form": {
-            "schema": {
-              "approver_comment": {
-                "type": "string"
-              }
-            }
-          }
-        },
-        "value": {
-          "type": "rawscript"
-        }
-      },
-      {
-        "id": "finalize_purchase",
-        "value": {
-          "type": "rawscript"
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "requester_email": {
-        "type": "string"
-      },
-      "amount": {
-        "type": "number"
-      }
-    },
-    "required": [
-      "requester_email",
-      "amount"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json

@@ -1,39 +0,0 @@
-{
-  "value": {
-    "modules": [
-      {
-        "id": "sum_numbers",
-        "value": {
-          "type": "script",
-          "path": "f/evals/add_two_numbers.ts",
-          "input_transforms": {
-            "a": {
-              "type": "javascript",
-              "expr": "flow_input.a"
-            },
-            "b": {
-              "type": "javascript",
-              "expr": "flow_input.b"
-            }
-          }
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "a": {
-        "type": "number"
-      },
-      "b": {
-        "type": "number"
-      }
-    },
-    "required": [
-      "a",
-      "b"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json

@@ -1,39 +0,0 @@
-{
-  "value": {
-    "modules": [
-      {
-        "id": "call_add_numbers",
-        "value": {
-          "type": "flow",
-          "path": "f/evals/add_numbers_flow",
-          "input_transforms": {
-            "a": {
-              "type": "javascript",
-              "expr": "flow_input.a"
-            },
-            "b": {
-              "type": "javascript",
-              "expr": "flow_input.b"
-            }
-          }
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "a": {
-        "type": "number"
-      },
-      "b": {
-        "type": "number"
-      }
-    },
-    "required": [
-      "a",
-      "b"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/expected/test3_branchone_routing.json

@@ -1,24 +0,0 @@
-{
-  "value": {
-    "modules": [
-      {
-        "id": "route_by_tier",
-        "value": {
-          "type": "branchone"
-        }
-      }
-    ]
-  },
-  "schema": {
-    "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "type": "object",
-    "properties": {
-      "tier": {
-        "type": "string"
-      }
-    },
-    "required": [
-      "tier"
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json

@@ -1,29 +0,0 @@
-{
-  "workspace": {
-    "scripts": [
-      {
-        "path": "f/evals/add_two_numbers.ts",
-        "summary": "Add two numbers",
-        "description": "Returns the sum of two numeric inputs.",
-        "language": "bun",
-        "schema": {
-          "$schema": "https://json-schema.org/draft/2020-12/schema",
-          "type": "object",
-          "properties": {
-            "a": {
-              "type": "number"
-            },
-            "b": {
-              "type": "number"
-            }
-          },
-          "required": [
-            "a",
-            "b"
-          ]
-        },
-        "content": "export async function main(a: number, b: number) {\n  return a + b;\n}\n"
-      }
-    ]
-  }
-}

ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json

@@ -1,49 +0,0 @@
-{
-  "workspace": {
-    "flows": [
-      {
-        "path": "f/evals/add_numbers_flow",
-        "summary": "Add two numbers in a subflow",
-        "description": "Takes two numeric inputs and returns their sum.",
-        "schema": {
-          "$schema": "https://json-schema.org/draft/2020-12/schema",
-          "type": "object",
-          "properties": {
-            "a": {
-              "type": "number"
-            },
-            "b": {
-              "type": "number"
-            }
-          },
-          "required": [
-            "a",
-            "b"
-          ]
-        },
-        "value": {
-          "modules": [
-            {
-              "id": "sum_numbers",
-              "value": {
-                "type": "rawscript",
-                "language": "bun",
-                "content": "export async function main(a: number, b: number) {\n  return a + b;\n}",
-                "input_transforms": {
-                  "a": {
-                    "type": "javascript",
-                    "expr": "flow_input.a"
-                  },
-                  "b": {
-                    "type": "javascript",
-                    "expr": "flow_input.b"
-                  }
-                }
-              }
-            }
-          ]
-        }
-      }
-    ]
-  }
-}

ai_evals/fixtures/frontend/script/expected/test1_greet_user.json

@@ -1,8 +0,0 @@
-{
-  "path": "f/evals/greet_user.ts",
-  "lang": "bun",
-  "args": {
-    "name": "Alice"
-  },
-  "code": "export async function main(name: string) {\n\treturn `Hello, ${name}!`\n}\n"
-}

ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json

@@ -1,8 +0,0 @@
-{
-  "path": "f/evals/greet_user.ts",
-  "lang": "bun",
-  "args": {
-    "name": "Alice"
-  },
-  "code": "export async function main(name: string) {\n\treturn ''\n}\n"
-}

ai_evals/history/.gitkeep

@@ -1 +0,0 @@
-

ai_evals/history/flow.jsonl

@@ -1,2 +0,0 @@
-{"createdAt":"2026-04-09T13:56:20.263Z","gitSha":"431c1d7f75c6f5c90f063dd18420b83a7b121353","mode":"flow","runs":1,"runModel":"anthropic:claude-haiku-4-5-20251001","judgeModel":"claude-sonnet-4-6","caseCount":13,"attemptCount":13,"passedAttempts":9,"passRate":0.6923076923076923,"averageDurationMs":34728.46153846154,"failedCaseIds":["flow-test4-order-processing-loop","flow-test6-ai-agent-tools","flow-test7-simple-modification","flow-test11-preprocessor-and-failure-handler"]}
-{"createdAt":"2026-04-09T13:58:53.544Z","gitSha":"47044b163510bf0b51d2ad2d8cf984c51f478415","mode":"flow","runs":1,"runModel":"anthropic:claude-haiku-4-5-20251001","judgeModel":"claude-sonnet-4-6","caseCount":13,"attemptCount":13,"passedAttempts":10,"passRate":0.7692307692307693,"averageDurationMs":30970.46153846154,"failedCaseIds":["flow-test6-ai-agent-tools","flow-test7-simple-modification","flow-test11-preprocessor-and-failure-handler"]}

ai_evals/modes/app.ts

@@ -1,78 +0,0 @@
-import { loadAppFixture } from "../adapters/frontend/core/app/appFixtureLoader";
-import type { AppFiles } from "../../frontend/src/lib/components/copilot/chat/app/core";
-import type { FrontendEvalModelConfig } from "../core/models";
-import { validateAppState, type AppFilesState } from "../core/validators";
-import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
-import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner";
-import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
-
-export function createAppModeRunner(
-  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
-): ModeRunner<AppFilesState, AppFilesState, AppFilesState> {
-  return {
-    mode: "app",
-    concurrency: 5,
-    judgeThreshold: 80,
-    async loadInitial(path) {
-      return path ? (await loadAppFixture(path)) : undefined;
-    },
-    async loadExpected(path) {
-      return path ? (await loadAppFixture(path)) : undefined;
-    },
-    async run(prompt, initial, context) {
-      const result = await runAppEval(prompt, getFrontendApiKey(modelConfig.provider), {
-        initialFrontend: initial?.frontend,
-        initialBackend: initial?.backend as AppFiles["backend"] | undefined,
-        provider: modelConfig.provider,
-        model: modelConfig.model,
-        runContext: context,
-      });
-
-      return {
-        success: result.success,
-        actual: result.files as AppFilesState,
-        error: result.error,
-        assistantMessageCount: result.assistantMessageCount,
-        toolCallCount: result.toolCallCount,
-        toolsUsed: result.toolsUsed,
-        skillsInvoked: [],
-      };
-    },
-    validate({ actual, initial, expected }) {
-      return validateAppState({ actual, initial, expected });
-    },
-    buildArtifacts(actual): BenchmarkArtifactFile[] {
-      const artifacts: BenchmarkArtifactFile[] = [
-        {
-          path: "app.json",
-          content: JSON.stringify(actual, null, 2) + "\n",
-        },
-      ];
-
-      for (const [filePath, content] of Object.entries(actual.frontend)) {
-        artifacts.push({
-          path: `frontend${filePath.startsWith("/") ? filePath : `/${filePath}`}`,
-          content,
-        });
-      }
-
-      for (const [key, runnable] of Object.entries(actual.backend)) {
-        artifacts.push({
-          path: `backend/${key}/meta.json`,
-          content: JSON.stringify(runnable, null, 2) + "\n",
-        });
-
-        const inlineContent = runnable.inlineScript?.content;
-        if (inlineContent) {
-          const extension = runnable.inlineScript?.language === "python3" ? "py" : "ts";
-          artifacts.push({
-            path: `backend/${key}/main.${extension}`,
-            content: inlineContent,
-          });
-        }
-      }
-
-      return artifacts;
-    },
-  };
-}

ai_evals/modes/cli.ts

@@ -1,156 +0,0 @@
-import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import path from "node:path";
-import { dirname, join } from "node:path";
-import { readFile } from "node:fs/promises";
-import { writeAiGuidanceFiles } from "../../cli/src/guidance/writer.ts";
-import type { CliEvalModelConfig } from "../core/models";
-import {
-  DEFAULT_CLI_EVAL_MODEL,
-  formatCliRunModelLabel,
-  getGeneratedSkillsSource,
-  runPromptAndCapture,
-} from "../adapters/cli/runtime";
-import { copyDirectory, readDirectoryFiles } from "../core/files";
-import { validateCliWorkspace } from "../core/validators";
-import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
-
-const IGNORE_WORKSPACE_FILES = new Set([".claude", "AGENTS.md", "CLAUDE.md", "rt.d.ts"]);
-
-interface CliWorkspaceFixture {
-  sourceDir: string;
-  files: Record<string, string>;
-}
-
-interface CliRunActual {
-  assistantOutput: string;
-  workspaceFiles: Record<string, string>;
-}
-
-const CLAUDE_PROJECT_PREAMBLE = [
-  "Follow the project instructions from AGENTS.md exactly.",
-  "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
-  "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
-  "Do not skip the Skill step.",
-].join(" ");
-
-export function createCliModeRunner(
-  modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
-): ModeRunner<CliWorkspaceFixture, CliWorkspaceFixture, CliRunActual> {
-  return {
-    mode: "cli",
-    concurrency: 1,
-    judgeThreshold: 80,
-    async loadInitial(path) {
-      return path
-        ? {
-            sourceDir: path,
-            files: await readDirectoryFiles(path),
-          }
-        : undefined;
-    },
-    async loadExpected(path) {
-      return path
-        ? {
-            sourceDir: path,
-            files: await readDirectoryFiles(path),
-          }
-        : undefined;
-    },
-    async run(prompt, initial, _context) {
-      const workspaceDir = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-"));
-
-      try {
-        if (initial) {
-          await copyDirectory(initial.sourceDir, workspaceDir);
-        }
-        await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true });
-        await writeAiGuidanceFiles({
-          targetDir: workspaceDir,
-          nonDottedPaths: true,
-          overwriteProjectGuidance: true,
-          skillsSourcePath: getGeneratedSkillsSource(),
-        });
-        await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
-
-        const renderedPrompt = await renderPrompt(prompt, workspaceDir);
-        const run = await runPromptAndCapture(renderedPrompt, workspaceDir, 6, modelConfig);
-        const workspaceFiles = await readDirectoryFiles(workspaceDir, { ignore: IGNORE_WORKSPACE_FILES });
-
-        return {
-          success: true,
-          actual: {
-            assistantOutput: run.output,
-            workspaceFiles,
-          },
-          assistantMessageCount: run.assistantMessageCount,
-          toolCallCount: run.toolsUsed.length,
-          toolsUsed: run.toolsUsed.map((entry) => entry.tool),
-          skillsInvoked: run.skillsInvoked,
-        };
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        return {
-          success: false,
-          actual: {
-            assistantOutput: "",
-            workspaceFiles: {},
-          },
-          error: message,
-          assistantMessageCount: 0,
-          toolCallCount: 0,
-          toolsUsed: [],
-          skillsInvoked: [],
-        };
-      } finally {
-        await rm(workspaceDir, { recursive: true, force: true });
-      }
-    },
-    validate({ actual, initial, expected }) {
-      return validateCliWorkspace({
-        actualFiles: actual.workspaceFiles,
-        expectedFiles: expected?.files,
-        initialFiles: initial?.files,
-      });
-    },
-    buildArtifacts(actual): BenchmarkArtifactFile[] {
-      const artifacts: BenchmarkArtifactFile[] = [
-        {
-          path: "assistant-output.txt",
-          content: `${actual.assistantOutput}\n`,
-        },
-      ];
-
-      for (const [filePath, content] of Object.entries(actual.workspaceFiles)) {
-        artifacts.push({
-          path: filePath,
-          content,
-        });
-      }
-
-      return artifacts;
-    },
-  };
-}
-
-export function getCliRunModelLabel(
-  modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
-): string {
-  return formatCliRunModelLabel(modelConfig);
-}
-
-async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
-  const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
-  const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8");
-
-  return [
-    "# Project Instructions",
-    agentsInstructions.trim(),
-    "",
-    "# Benchmark Harness",
-    CLAUDE_PROJECT_PREAMBLE,
-    "",
-    "# User Request",
-    renderedUserPrompt,
-  ].join("\n");
-}

ai_evals/modes/flow.ts

@@ -1,105 +0,0 @@
-import { readJsonFile } from "../core/files";
-import type { FrontendEvalModelConfig } from "../core/models";
-import { validateFlowState, type FlowState } from "../core/validators";
-import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
-import {
-  runFlowEval,
-  type FlowFixture,
-} from "../adapters/frontend/core/flow/flowEvalRunner";
-import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers";
-import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
-
-interface FlowInitialFixture {
-  flow?: FlowFixture;
-  workspace?: FlowWorkspaceFixtures;
-}
-
-export function createFlowModeRunner(
-  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
-): ModeRunner<FlowInitialFixture, FlowState, FlowState> {
-  return {
-    mode: "flow",
-    concurrency: 5,
-    judgeThreshold: 80,
-    async loadInitial(path) {
-      if (!path) {
-        return undefined;
-      }
-      return normalizeFlowInitialFixture(await readJsonFile<unknown>(path));
-    },
-    async loadExpected(path) {
-      if (!path) {
-        return undefined;
-      }
-      return normalizeFlowStateFixture(await readJsonFile<unknown>(path));
-    },
-    async run(prompt, initial, context) {
-      const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), {
-        initialFlow: initial?.flow,
-        workspaceFixtures: initial?.workspace,
-        provider: modelConfig.provider,
-        model: modelConfig.model,
-        runContext: context,
-      });
-
-      return {
-        success: result.success,
-        actual: {
-          value: { modules: result.flow.value?.modules ?? [] },
-          schema: result.flow.schema,
-        },
-        error: result.error,
-        assistantMessageCount: result.assistantMessageCount,
-        toolCallCount: result.toolCallCount,
-        toolsUsed: result.toolsUsed,
-        skillsInvoked: [],
-      };
-    },
-    validate({ evalCase, actual, initial }) {
-      return validateFlowState({
-        actual,
-        initial: initial?.flow,
-        validate: evalCase.validate,
-      });
-    },
-    buildArtifacts(actual): BenchmarkArtifactFile[] {
-      return [
-        {
-          path: "flow.json",
-          content: JSON.stringify(actual, null, 2) + "\n",
-        },
-      ];
-    },
-  };
-}
-
-function normalizeFlowInitialFixture(value: unknown): FlowInitialFixture {
-  if (isObject(value) && ("flow" in value || "workspace" in value)) {
-    const fixture = value as {
-      flow?: FlowFixture;
-      workspace?: FlowWorkspaceFixtures;
-    };
-    return {
-      flow: fixture.flow,
-      workspace: fixture.workspace,
-    };
-  }
-
-  return {
-    flow: normalizeFlowStateFixture(value),
-  };
-}
-
-function normalizeFlowStateFixture(value: unknown): FlowState {
-  if (!isObject(value)) {
-    return {};
-  }
-  if ("flow" in value && isObject((value as { flow?: unknown }).flow)) {
-    return (value as { flow: FlowState }).flow;
-  }
-  return value as FlowState;
-}
-
-function isObject(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}

ai_evals/modes/frontendCommon.ts

@@ -1,23 +0,0 @@
-import {
-  getFrontendEvalModel,
-  resolveEvalModel,
-  type FrontendEvalModelConfig,
-} from "../core/models";
-
-export const DEFAULT_FRONTEND_EVAL_MODEL: FrontendEvalModelConfig = getFrontendEvalModel(
-  resolveEvalModel("flow")
-);
-
-export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string {
-  const apiKey =
-    provider === "anthropic" ? process.env.ANTHROPIC_API_KEY : process.env.OPENAI_API_KEY;
-  if (!apiKey) {
-    const envName = provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY";
-    throw new Error(`${envName} is required for frontend evals`);
-  }
-  return apiKey;
-}
-
-export function getFrontendRunModelLabel(model: FrontendEvalModelConfig): string {
-  return `${model.provider}:${model.model}`;
-}

ai_evals/modes/script.ts

@@ -1,60 +0,0 @@
-import { readJsonFile } from "../core/files";
-import type { FrontendEvalModelConfig } from "../core/models";
-import { validateScriptState } from "../core/validators";
-import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
-import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
-import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
-import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
-
-export function createScriptModeRunner(
-  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
-): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
-  return {
-    mode: "script",
-    concurrency: 5,
-    judgeThreshold: 80,
-    async loadInitial(path) {
-      return path ? await readJsonFile<ScriptEvalState>(path) : undefined;
-    },
-    async loadExpected(path) {
-      return path ? await readJsonFile<ScriptEvalState>(path) : undefined;
-    },
-    async run(prompt, initial, context) {
-      if (!initial) {
-        throw new Error("Script evals require an initial script fixture");
-      }
-
-      const result = await runScriptEval(prompt, getFrontendApiKey(modelConfig.provider), {
-        initialScript: initial,
-        provider: modelConfig.provider,
-        model: modelConfig.model,
-        runContext: context,
-      });
-
-      return {
-        success: result.success,
-        actual: result.script,
-        error: result.error,
-        assistantMessageCount: result.assistantMessageCount,
-        toolCallCount: result.toolCallCount,
-        toolsUsed: result.toolsUsed,
-        skillsInvoked: [],
-      };
-    },
-    validate({ actual, initial, expected }) {
-      return validateScriptState({ actual, initial, expected });
-    },
-    buildArtifacts(actual): BenchmarkArtifactFile[] {
-      return [
-        {
-          path: "script.json",
-          content: JSON.stringify(actual, null, 2) + "\n",
-        },
-        {
-          path: actual.path,
-          content: actual.code,
-        },
-      ];
-    },
-  };
-}

ai_evals/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "windmill-ai-evals",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "cli": "bun cli/index.ts"
-  },
-  "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.2.25",
-    "@anthropic-ai/sdk": "^0.39.0",
-    "commander": "^14.0.3",
-    "yaml": "^2.8.3"
-  },
-  "devDependencies": {
-    "@types/bun": "latest",
-    "typescript": "^5.0.0"
-  }
-}

backend/.sqlx/query-05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2",
+  "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -57,11 +57,6 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
-      },
-      {
-        "ordinal": 11,
-        "name": "disabled",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -81,9 +76,8 @@
       true,
       null,
       false,
-      false,
       false
     ]
   },
-  "hash": "115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc"
+  "hash": "05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab"
 }

backend/.sqlx/query-08522e494e34f4ecae21460262bf0ed3c5a197dd744c87cb760aaf47001febbd.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM trashbin WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "08522e494e34f4ecae21460262bf0ed3c5a197dd744c87cb760aaf47001febbd"
-}

backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "exists!",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d"
-}

backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM token WHERE email = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6"
-}

backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            external_id,\n            workspace_id,\n            service_name AS \"service_name!: ServiceName\",\n            script_path,\n            is_flow,\n            webhook_token_hash,\n            service_config,\n            error,\n            created_at,\n            updated_at,\n            summary\n        FROM\n            native_trigger\n        WHERE\n            workspace_id = $1\n            AND service_name = $2\n            AND script_path = $3\n            AND is_flow = $4\n        LIMIT 1\n        ",
+  "query": "\n        SELECT\n            external_id,\n            workspace_id,\n            service_name AS \"service_name!: ServiceName\",\n            script_path,\n            is_flow,\n            webhook_token_hash,\n            service_config,\n            error,\n            created_at,\n            updated_at\n        FROM\n            native_trigger\n        WHERE\n            workspace_id = $1\n            AND service_name = $2\n            AND script_path = $3\n            AND is_flow = $4\n        LIMIT 1\n        ",
   "describe": {
     "columns": [
       {
@@ -62,11 +62,6 @@
         "ordinal": 9,
         "name": "updated_at",
         "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 10,
-        "name": "summary",
-        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -97,9 +92,8 @@
       true,
       true,
       false,
-      false,
-      true
+      false
     ]
   },
-  "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa"
+  "hash": "1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce"
 }

backend/.sqlx/query-1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef.json

@@ -1,28 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT ws.default_app AS default_app_path, av.raw_app AS \"default_app_raw: Option<bool>\"\n         FROM workspace_settings ws\n         LEFT JOIN app ON app.path = ws.default_app AND app.workspace_id = ws.workspace_id\n         LEFT JOIN app_version av ON av.id = app.versions[array_upper(app.versions, 1)]\n         WHERE ws.workspace_id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "default_app_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "default_app_raw: Option<bool>",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      false
-    ]
-  },
-  "hash": "1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef"
-}

backend/.sqlx/query-1d995dd5a094631ae96c16d68026fdeb22714af38162e87c02b052a5b8ec2645.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM trashbin WHERE workspace_id = $1 AND id = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "1d995dd5a094631ae96c16d68026fdeb22714af38162e87c02b052a5b8ec2645"
-}

backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json

@@ -1,32 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n            schemaname || '.' || relname as \"table_name!\",\n            pg_total_relation_size(relid) as \"total_size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n        FROM pg_catalog.pg_statio_user_tables\n        ORDER BY pg_total_relation_size(relid) DESC\n        LIMIT 15",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "table_name!",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "total_size_bytes!",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 2,
-        "name": "total_size_pretty!",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null,
-      null,
-      null
-    ]
-  },
-  "hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6"
-}

backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af"
-}

backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json

@@ -1,28 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT email, disabled FROM password WHERE email = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "disabled",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false
-    ]
-  },
-  "hash": "23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443"
-}

backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "value",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2"
-}

backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json

@@ -1,32 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n            ws.workspace_id as \"workspace_id!\",\n            dt.key as \"name!\",\n            dt.value->>'table_name' as \"table_name\"\n        FROM workspace_settings ws,\n             jsonb_each(ws.datatable) dt\n        WHERE dt.value->>'resource_type' = 'instance'\n          AND dt.value->>'table_name' IS NOT NULL",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "workspace_id!",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "name!",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "table_name",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      null,
-      null
-    ]
-  },
-  "hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720"
-}

backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "flow_step_id",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true,
+      true
+    ]
+  },
+  "hash": "32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9"
+}

backend/.sqlx/query-3453c0b7dd3c4d2c9bc639f379901741955502c9345e82a9b7fbbf3d3c7ab517.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM trashbin WHERE expires_at <= now()",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "3453c0b7dd3c4d2c9bc639f379901741955502c9345e82a9b7fbbf3d3c7ab517"
-}

backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "size_bytes",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "size_pretty",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null,
-      null
-    ]
-  },
-  "hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0"
-}

backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        UPDATE native_trigger\n        SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, summary = $8, error = NULL, updated_at = NOW()\n        WHERE\n            workspace_id = $5\n            AND service_name = $6\n            AND external_id = $7\n        ",
+  "query": "\n        UPDATE native_trigger\n        SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, error = NULL, updated_at = NOW()\n        WHERE\n            workspace_id = $5\n            AND service_name = $6\n            AND external_id = $7\n        ",
   "describe": {
     "columns": [],
     "parameters": {
@@ -21,11 +21,10 @@
             }
           }
         },
-        "Text",
-        "Varchar"
+        "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708"
+  "hash": "40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e"
 }

backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json

@@ -1,65 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, workspace_id, item_kind, item_path, item_data, deleted_by, deleted_at, expires_at\n         FROM trashbin\n         WHERE workspace_id = $1 AND id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "item_kind",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "item_path",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "item_data",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 5,
-        "name": "deleted_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 6,
-        "name": "deleted_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 7,
-        "name": "expires_at",
-        "type_info": "Timestamptz"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7"
-}

backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json

@@ -1,61 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, workspace_id, item_kind, item_path, deleted_by, deleted_at, expires_at\n             FROM trashbin\n             WHERE workspace_id = $1 AND item_kind = $2\n             ORDER BY deleted_at DESC\n             LIMIT $3 OFFSET $4",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "item_kind",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "item_path",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "deleted_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "deleted_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 6,
-        "name": "expires_at",
-        "type_info": "Timestamptz"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef"
-}

backend/.sqlx/query-52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT DISTINCT imported_path as \"imported_path!\"\n            FROM dependency_map\n            WHERE workspace_id = $1\n                AND importer_path = $2\n                AND imported_path NOT LIKE 'dependencies/%'\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "imported_path!",
-        "type_info": "Varchar"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765"
-}

backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json

@@ -1,35 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT email, is_service_account, disabled FROM usr WHERE username = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "email",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "is_service_account",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 2,
-        "name": "disabled",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb"
-}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      true
+      null
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json

@@ -47,11 +47,6 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "is_service_account",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -68,8 +63,7 @@
       false,
       false,
       true,
-      true,
-      false
+      true
     ]
   },
   "hash": "5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c"

backend/.sqlx/query-60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n            authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n            SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled\n            FROM password\n            WHERE email IN (SELECT email FROM active_users)\n            ORDER BY super_admin DESC, devops DESC\n            LIMIT $1 OFFSET $2",
+  "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n            authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n            SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source\n            FROM password\n            WHERE email IN (SELECT email FROM active_users)\n            ORDER BY super_admin DESC, devops DESC\n            LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -57,11 +57,6 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
-      },
-      {
-        "ordinal": 11,
-        "name": "disabled",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -81,9 +76,8 @@
       true,
       true,
       false,
-      false,
       false
     ]
   },
-  "hash": "a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10"
+  "hash": "60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce"
 }

backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json

@@ -47,11 +47,6 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 9,
-        "name": "is_service_account",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -69,8 +64,7 @@
       false,
       false,
       true,
-      true,
-      false
+      true
     ]
   },
   "hash": "60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b"

backend/.sqlx/query-638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT (email) DO UPDATE SET disabled = false",
+  "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT DO NOTHING",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656"
+  "hash": "638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6"
 }

backend/.sqlx/query-65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE email = $1",
+  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password WHERE email = $1",
   "describe": {
     "columns": [
       {
@@ -57,11 +57,6 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
-      },
-      {
-        "ordinal": 11,
-        "name": "disabled",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -80,9 +75,8 @@
       true,
       null,
       false,
-      false,
       false
     ]
   },
-  "hash": "f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493"
+  "hash": "65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9"
 }

backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND (username = $2 OR email = $3))",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "exists",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040"
-}

backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
+  "query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
   "describe": {
     "columns": [
       {
@@ -50,16 +50,11 @@
       },
       {
         "ordinal": 9,
-        "name": "is_service_account",
+        "name": "super_admin",
         "type_info": "Bool"
       },
       {
         "ordinal": 10,
-        "name": "super_admin!",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 11,
         "name": "name",
         "type_info": "Varchar"
       }
@@ -81,9 +76,8 @@
       true,
       true,
       false,
-      null,
       true
     ]
   },
-  "hash": "1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1"
+  "hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
 }

backend/.sqlx/query-6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT pg_advisory_xact_lock(hashtext($1 || '/' || $2))",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "pg_advisory_xact_lock",
-        "type_info": "Void"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be"
-}

backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        INSERT INTO native_trigger (\n            external_id,\n            workspace_id,\n            service_name,\n            script_path,\n            is_flow,\n            webhook_token_hash,\n            service_config,\n            summary\n        ) VALUES (\n            $1, $2, $3, $4, $5, $6, $7, $8\n        )\n        ON CONFLICT (external_id, workspace_id, service_name)\n        DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n        ",
+  "query": "\n        INSERT INTO native_trigger (\n            external_id,\n            workspace_id,\n            service_name,\n            script_path,\n            is_flow,\n            webhook_token_hash,\n            service_config\n        ) VALUES (\n            $1, $2, $3, $4, $5, $6, $7\n        )\n        ON CONFLICT (external_id, workspace_id, service_name)\n        DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, error = NULL, updated_at = NOW()\n        ",
   "describe": {
     "columns": [],
     "parameters": {
@@ -21,11 +21,10 @@
         "Varchar",
         "Bool",
         "Varchar",
-        "Jsonb",
-        "Varchar"
+        "Jsonb"
       ]
     },
     "nullable": []
   },
-  "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e"
+  "hash": "6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7"
 }

backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
   "describe": {
     "columns": [
       {
@@ -12,11 +12,6 @@
         "ordinal": 1,
         "name": "deployment_msg",
         "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "created_at",
-        "type_info": "Timestamptz"
       }
     ],
     "parameters": {
@@ -27,9 +22,8 @@
     },
     "nullable": [
       false,
-      true,
-      false
+      true
     ]
   },
-  "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c"
+  "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de"
 }

backend/.sqlx/query-79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        WITH dk AS (\n            INSERT INTO debounce_key (job_id, key)\n            VALUES ($1, $2)\n            ON CONFLICT (key)\n            DO UPDATE SET\n                previous_job_id = debounce_key.job_id,\n                job_id = EXCLUDED.job_id,\n                debounced_times = debounce_key.debounced_times + 1\n            RETURNING\n                debounced_times,\n                first_started_at,\n                previous_job_id AS job_id_to_debounce\n        ), _batch AS (\n            INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n            SELECT\n                $1,\n                COALESCE(\n                    (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n                    nextval('debounce_batch_seq')\n                )\n            FROM dk\n        )\n        SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "debounced_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "first_started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 2,
+        "name": "job_id_to_debounce",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073"
+}

backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json

@@ -1,40 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT\n            c.relname as \"table_name!\",\n            pg_total_relation_size(c.oid) as \"size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n            COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n        FROM pg_class c\n        JOIN pg_namespace n ON n.oid = c.relnamespace\n        WHERE n.nspname = 'public' AND c.relname = ANY($1)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "table_name!",
-        "type_info": "Name"
-      },
-      {
-        "ordinal": 1,
-        "name": "size_bytes!",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 2,
-        "name": "size_pretty!",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 3,
-        "name": "estimated_rows!",
-        "type_info": "Float4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "NameArray"
-      ]
-    },
-    "nullable": [
-      false,
-      null,
-      null,
-      null
-    ]
-  },
-  "hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84"
-}

backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "cnt",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9"
-}

backend/.sqlx/query-87ee10d8ba5ba281781f23e5390190fb90df980a19c900452f9b1a19c3620e30.json

@@ -1,32 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                        SELECT\n                            route_path,\n                            workspace_id,\n                            http_method::TEXT AS \"http_method!\"\n                        FROM\n                            http_trigger\n                        WHERE\n                            workspaced_route IS FALSE\n                            AND route_path_key IN (\n                                SELECT\n                                    route_path_key\n                                FROM\n                                    http_trigger\n                                WHERE\n                                    workspaced_route IS FALSE\n                                GROUP BY\n                                    route_path_key, http_method\n                                HAVING COUNT(*) > 1\n                            )\n                        ORDER BY route_path_key\n                    ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "route_path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "http_method!",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      null
-    ]
-  },
-  "hash": "87ee10d8ba5ba281781f23e5390190fb90df980a19c900452f9b1a19c3620e30"
-}

backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO trashbin (workspace_id, item_kind, item_path, item_data, deleted_by)\n         VALUES ($1, $2, $3, $4, $5) RETURNING id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Text",
-        "Jsonb",
-        "Varchar"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f"
-}

backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE password SET disabled = $1 WHERE email = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Bool",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67"
-}

backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json

@@ -1,60 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, workspace_id, item_kind, item_path, deleted_by, deleted_at, expires_at\n             FROM trashbin\n             WHERE workspace_id = $1\n             ORDER BY deleted_at DESC\n             LIMIT $2 OFFSET $3",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "workspace_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "item_kind",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 3,
-        "name": "item_path",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "deleted_by",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 5,
-        "name": "deleted_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 6,
-        "name": "expires_at",
-        "type_info": "Timestamptz"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32"
-}

backend/.sqlx/query-950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json

@@ -0,0 +1,46 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        WITH RECURSIVE chain AS (\n            SELECT\n                j.id,\n                j.parent_job,\n                j.flow_step_id,\n                1 AS depth\n            FROM v2_job j\n            WHERE j.id = $1\n          UNION ALL\n            SELECT\n                pj.id,\n                pj.parent_job,\n                pj.flow_step_id,\n                c.depth + 1\n            FROM chain c\n            JOIN v2_job pj ON pj.id = c.parent_job\n            WHERE c.parent_job IS NOT NULL\n        )\n        SELECT\n            c.id,\n            c.parent_job,\n            c.flow_step_id,\n            EXISTS(SELECT 1 FROM v2_job_queue q WHERE q.id = c.parent_job) AS \"parent_in_queue!\",\n            EXISTS(\n                SELECT 1 FROM v2_job sib\n                WHERE sib.parent_job = c.parent_job\n                  AND sib.id != c.id\n                  AND sib.id IN (SELECT sq.id FROM v2_job_queue sq)\n            ) AS \"has_other_active_siblings!\"\n        FROM chain c\n        WHERE c.depth >= 1\n        ORDER BY c.depth ASC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 2,
+        "name": "flow_step_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "parent_in_queue!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "has_other_active_siblings!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb"
+}

backend/.sqlx/query-975099ff6b07718ea94bcb5f84a4414c59199964cee53ef2ee6b35a78cf0c49a.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT value FROM global_settings WHERE name = 'ai_config'",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "value",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "975099ff6b07718ea94bcb5f84a4414c59199964cee53ef2ee6b35a78cf0c49a"
-}

backend/.sqlx/query-98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                  WITH dk AS (\n                      INSERT INTO debounce_key (job_id, key)\n                      VALUES ($1, $2)\n                      ON CONFLICT (key)\n                      DO UPDATE SET\n                          previous_job_id = debounce_key.job_id,\n                          job_id = EXCLUDED.job_id,\n                          debounced_times = debounce_key.debounced_times + 1\n                      RETURNING\n                          debounced_times,\n                          first_started_at,\n                          previous_job_id AS job_id_to_debounce\n                  ), _batch AS (\n                      INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n                      SELECT\n                          $1,\n                          COALESCE(\n                              (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n                              nextval('debounce_batch_seq')\n                          )\n                      FROM dk\n                  )\n                  SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "debounced_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "first_started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 2,
+        "name": "job_id_to_debounce",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9"
+}