Merge branch 'main' into py-typechecked-decorator

* chore(main): release 1.668.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>

.github/workflows/benchmark.yml

@@ -290,6 +290,49 @@ jobs:
           path: |
             *.json
 
+  benchmark_wac:
+    runs-on: ubicloud-standard-8
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+          --shm-size=2g
+      windmill:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+    steps:
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      - name: benchmark
+        timeout-minutes: 30
+        run: deno run  -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_wac.json
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_wac
+          path: |
+            *.json
+
   benchmark_graphs:
     runs-on: ubicloud
     needs:
@@ -297,6 +340,7 @@ jobs:
       - benchmark_dedicated
       - benchmark_4workers
       - benchmark_8workers
+      - benchmark_wac
     steps:
       - uses: denoland/setup-deno@v2
         with:

.github/workflows/cli-tests.yml

@@ -1,6 +1,7 @@
 name: CLI Tests
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:

.github/workflows/rust-client-check.yml

@@ -18,10 +18,7 @@ jobs:
     runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v20
-        with:
-          extra_nix_config: |
-            experimental-features = nix-command flakes
+      - uses: cachix/install-nix-action@v31
       - name: Check rust client builds
         run: cd rust-client && nix develop ../ --command ./dev.nu --check
         timeout-minutes: 16

.github/workflows/rust_on_release.yml

@@ -10,10 +10,7 @@ jobs:
     runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v20
-        with:
-          extra_nix_config: |
-            experimental-features = nix-command flakes
+      - uses: cachix/install-nix-action@v31
       - run: cd rust-client && nix develop ../ --command ./dev.nu --check --publish 
         env:
           CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.webmux.yaml

@@ -55,11 +55,13 @@ profiles:
       - id: backend
         kind: command
         split: right
-        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/backend" && cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
+        workingDir: backend
+        command: PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
       - id: frontend
         kind: command
         split: bottom
-        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0
+        workingDir: frontend
+        command: npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
         
   frontendOnly:
     runtime: host
@@ -82,7 +84,8 @@ profiles:
       - id: frontend
         kind: command
         split: right
-        command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0
+        workingDir: frontend
+        command: npm run generate-backend-client && npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
 
   agentOnly:
     runtime: host

CHANGELOG.md

@@ -1,5 +1,106 @@
 # Changelog
 
+## [1.668.2](https://github.com/windmill-labs/windmill/compare/v1.668.1...v1.668.2) (2026-03-28)
+
+
+### Bug Fixes
+
+* **cli:** app push crash, lint path, push --message, run validation, history timestamps ([#8585](https://github.com/windmill-labs/windmill/issues/8585)) ([f40cdaf](https://github.com/windmill-labs/windmill/commit/f40cdaf43453d2643800ed730d6abe6873bbe8e7))
+
+## [1.668.1](https://github.com/windmill-labs/windmill/compare/v1.668.0...v1.668.1) (2026-03-28)
+
+
+### Bug Fixes
+
+* **cli:** fix 13 CLI bugs — exit codes, sync tar fallback, variable encryption, JSON output ([#8582](https://github.com/windmill-labs/windmill/issues/8582)) ([38acaa3](https://github.com/windmill-labs/windmill/commit/38acaa3653728bf9e0ae6f746edf433703b4ab63))
+
+## [1.668.0](https://github.com/windmill-labs/windmill/compare/v1.667.0...v1.668.0) (2026-03-28)
+
+
+### Features
+
+* add DB health diagnostic dashboard for superadmins ([#8574](https://github.com/windmill-labs/windmill/issues/8574)) ([9ceab73](https://github.com/windmill-labs/windmill/commit/9ceab730d7def09c2b46527f8a586789d14f2ce0))
+* **cli:** add job, group, audit, token commands and schedule enable/disable ([#8581](https://github.com/windmill-labs/windmill/issues/8581)) ([d29cb23](https://github.com/windmill-labs/windmill/commit/d29cb234dbff07473b911e5e75e362def8a47650))
+* IAM RDS auth for PostgreSQL worker resources ([#8573](https://github.com/windmill-labs/windmill/issues/8573)) ([56253c0](https://github.com/windmill-labs/windmill/commit/56253c04cb679c58d00750da699a6cb62ed52aca))
+
+
+### Bug Fixes
+
+* add Authority Key Identifier to MITM proxy leaf certs ([#8576](https://github.com/windmill-labs/windmill/issues/8576)) ([ce2e6c8](https://github.com/windmill-labs/windmill/commit/ce2e6c8c015110d0385e6afecdc8313aabca1364))
+* Improve CLI developer experience: error handling, sync workflow, JSON output, workspace forks ([#8578](https://github.com/windmill-labs/windmill/issues/8578)) ([501a4ff](https://github.com/windmill-labs/windmill/commit/501a4ff2a94510145952686d24ccc639781beefe))
+* trigger capture filter and focus issues ([#8579](https://github.com/windmill-labs/windmill/issues/8579)) ([820f28f](https://github.com/windmill-labs/windmill/commit/820f28f8799f8dad5cfab94b51ac9921d664f04a))
+
+## [1.667.0](https://github.com/windmill-labs/windmill/compare/v1.666.0...v1.667.0) (2026-03-27)
+
+
+### Features
+
+* add schedule support to CLI branch-specific items ([#8570](https://github.com/windmill-labs/windmill/issues/8570)) ([b592996](https://github.com/windmill-labs/windmill/commit/b592996eee98ddb664f1b007b95a2096d5d4e3a6))
+* add workspace-level service accounts ([#8560](https://github.com/windmill-labs/windmill/issues/8560)) ([3959fe8](https://github.com/windmill-labs/windmill/commit/3959fe82974f5f0383e94fd83a5d78fe4212d56a))
+* **cli:** generate commented wmill.yaml and add config reference command ([#8546](https://github.com/windmill-labs/windmill/issues/8546)) ([d06b426](https://github.com/windmill-labs/windmill/commit/d06b42613f73c4a7b31c990be22b0c97efab2666))
+* DB-coordinated graceful restart staggering for settings changes ([#8555](https://github.com/windmill-labs/windmill/issues/8555)) ([2f32675](https://github.com/windmill-labs/windmill/commit/2f326758013dd1f1e6ae732e5784a32f1fb6e4bd))
+* improve-replay-ui ([#8250](https://github.com/windmill-labs/windmill/issues/8250)) ([c0aafee](https://github.com/windmill-labs/windmill/commit/c0aafee9a9923d5dc2fa3b99da4378e923933a06))
+* support multiple folder selection in MCP scope selector ([#8557](https://github.com/windmill-labs/windmill/issues/8557)) ([ad19ac9](https://github.com/windmill-labs/windmill/commit/ad19ac9b37b04591c921f93f180bdda961af6cef))
+
+
+### Bug Fixes
+
+* **cli:** preserve inline script files during flow generate-locks ([#8561](https://github.com/windmill-labs/windmill/issues/8561)) ([a8b651d](https://github.com/windmill-labs/windmill/commit/a8b651da9ff86766119e14c0b61652be8a7b453a))
+* emit 0 for OTEL queue metrics when tag queue is empty ([#8559](https://github.com/windmill-labs/windmill/issues/8559)) ([79cc4a9](https://github.com/windmill-labs/windmill/commit/79cc4a92d88486c999799826bd0c9663767103f5))
+* handle inline script deletion in sync push + flow new nonDottedPaths ([#8553](https://github.com/windmill-labs/windmill/issues/8553)) ([943fe9c](https://github.com/windmill-labs/windmill/commit/943fe9c6cc9b046e24007e45b5c37afc4804256a))
+* include importer_kind in dependency debounce key to prevent cross-kind collisions ([#8567](https://github.com/windmill-labs/windmill/issues/8567)) ([bc7007b](https://github.com/windmill-labs/windmill/commit/bc7007bb4265e1f1375c1f0678b74325882a4e92))
+* multi-script dedicated workers race on shared job_dir ([#8551](https://github.com/windmill-labs/windmill/issues/8551)) ([#8569](https://github.com/windmill-labs/windmill/issues/8569)) ([63a3573](https://github.com/windmill-labs/windmill/commit/63a3573951d1f724cc63728ed973d039a5468072))
+* preserve notes on nodes inside collapsed groups ([#8552](https://github.com/windmill-labs/windmill/issues/8552)) ([0fb1153](https://github.com/windmill-labs/windmill/commit/0fb115304afc49812420e9ce24e5048502621059))
+* sanitize flow step summaries for filesystem-safe names ([#8554](https://github.com/windmill-labs/windmill/issues/8554)) ([e15bfbf](https://github.com/windmill-labs/windmill/commit/e15bfbf91ee1517432a6861ebb48e129485006aa))
+* use admin db pool in get_copilot_settings_state ([#8564](https://github.com/windmill-labs/windmill/issues/8564)) ([70f3ee5](https://github.com/windmill-labs/windmill/commit/70f3ee5ed4470e9993be822874f2b38e83a96611))
+
+
+### Performance Improvements
+
+* enable bun bundle caching for WAC v2 scripts ([#8556](https://github.com/windmill-labs/windmill/issues/8556)) ([ab868e9](https://github.com/windmill-labs/windmill/commit/ab868e9ebceadaa55e54770d9d59dc5524da13ff))
+
+## [1.666.0](https://github.com/windmill-labs/windmill/compare/v1.665.0...v1.666.0) (2026-03-26)
+
+
+### Features
+
+* add PDF input support to AI agent ([#8525](https://github.com/windmill-labs/windmill/issues/8525)) ([e44504c](https://github.com/windmill-labs/windmill/commit/e44504c6e93e7a4ee94ced03ab626b79a4fd0754))
+
+
+### Bug Fixes
+
+* add relative imports to the dependency list in deploymentUI ([#8548](https://github.com/windmill-labs/windmill/issues/8548)) ([d760ea5](https://github.com/windmill-labs/windmill/commit/d760ea5eaf4dc33007f1fd3e5e07b86925a0aa11))
+* filter null entries in FileUpload initialValue to prevent s3 access error ([#8544](https://github.com/windmill-labs/windmill/issues/8544)) ([1a73012](https://github.com/windmill-labs/windmill/commit/1a73012e0737a6ebea8307013dc0f79982269d91))
+* pass pre-bound TcpListener to run_server to fix Windows CI test race ([#8542](https://github.com/windmill-labs/windmill/issues/8542)) ([d7f4b95](https://github.com/windmill-labs/windmill/commit/d7f4b950ce6e966ed1b410e03d48fe96bc036e73))
+* resolve parent_hash race condition in sync push with auto_parent ([#8545](https://github.com/windmill-labs/windmill/issues/8545)) ([71549c3](https://github.com/windmill-labs/windmill/commit/71549c3db053bcc209c7065ac8cd42f1e8047cc3))
+* upload_s3_file not working in VS Code extension ([#8547](https://github.com/windmill-labs/windmill/issues/8547)) ([1fa4d91](https://github.com/windmill-labs/windmill/commit/1fa4d919b30ac9eff2d1789fba2695450ba115e7))
+
+## [1.665.0](https://github.com/windmill-labs/windmill/compare/v1.664.0...v1.665.0) (2026-03-26)
+
+
+### Features
+
+* add instance setting to enforce workspace prefix for HTTP routes ([#8528](https://github.com/windmill-labs/windmill/issues/8528)) ([9b3e558](https://github.com/windmill-labs/windmill/commit/9b3e558d84f15052e9c32695a467f8ef7e4ad1f5))
+* add trashbin system for soft-deleting items ([#8519](https://github.com/windmill-labs/windmill/issues/8519)) ([69ce946](https://github.com/windmill-labs/windmill/commit/69ce946241d98ea90bc7135d44ca0c87f928be88))
+* mask sensitive values in job logs ([#8520](https://github.com/windmill-labs/windmill/issues/8520)) ([0885d8c](https://github.com/windmill-labs/windmill/commit/0885d8c986f13ac210e4db3ad38febe9be391ba4))
+* move basic git sync from EE to CE with runtime user count gating ([#8493](https://github.com/windmill-labs/windmill/issues/8493)) ([79d2bd5](https://github.com/windmill-labs/windmill/commit/79d2bd51a00654162754046308d7670242120df6))
+* runner groups for shared-process multi-script dedicated workers ([#8434](https://github.com/windmill-labs/windmill/issues/8434)) ([c28314f](https://github.com/windmill-labs/windmill/commit/c28314f424ea0e04b86565ce88e6c91e0df1a0cf))
+* SCIM user deprovisioning (active:false) + instance-level user disable ([#8484](https://github.com/windmill-labs/windmill/issues/8484)) ([0bd7568](https://github.com/windmill-labs/windmill/commit/0bd756839c0261f255111d62088bdaaecb838085))
+* show groups and notes in flow status viewer ([#8535](https://github.com/windmill-labs/windmill/issues/8535)) ([167084a](https://github.com/windmill-labs/windmill/commit/167084a0ebe73384fa0d31f0b24017a47686a072))
+
+
+### Bug Fixes
+
+* auto-generate datatable SDK reference for app mode system prompt ([#8522](https://github.com/windmill-labs/windmill/issues/8522)) ([8a32322](https://github.com/windmill-labs/windmill/commit/8a32322c187ccc60ec7eafb61a9678f267a82282))
+* consider wmill.yaml environments alias in git sync ([#8532](https://github.com/windmill-labs/windmill/issues/8532)) ([b7475c7](https://github.com/windmill-labs/windmill/commit/b7475c73094a28f520f798f6cb1a0c6b4807ccb7))
+* GitHub Enterprise Server support for self-managed GitHub Apps ([#8507](https://github.com/windmill-labs/windmill/issues/8507)) ([935fb44](https://github.com/windmill-labs/windmill/commit/935fb44c848b8bf9430b5600dd3c3bedb2f89efd))
+* raw apps bundle not found during deployment error ([#8515](https://github.com/windmill-labs/windmill/issues/8515)) ([34e3115](https://github.com/windmill-labs/windmill/commit/34e3115bcbd19a8e0b6f483435586a2ab43d0a8e))
+* require admin for workspace encryption key export ([#8523](https://github.com/windmill-labs/windmill/issues/8523)) ([0317668](https://github.com/windmill-labs/windmill/commit/031766808945aefc926f0836d011c0b2a5d2243d))
+* restrict logout redirect to whitelisted domains ([#8524](https://github.com/windmill-labs/windmill/issues/8524)) ([4c8edd5](https://github.com/windmill-labs/windmill/commit/4c8edd5e944d77ed2d41c2b87171c1115c0fdcdc))
+* serve index disk storage sizes from /srch/ endpoint ([#8511](https://github.com/windmill-labs/windmill/issues/8511)) ([e3620e0](https://github.com/windmill-labs/windmill/commit/e3620e074e1bdb46b2b8d732f35a91d300589663))
+* use /apps_raw/get/ redirect URL for raw apps set as workspace default ([#8508](https://github.com/windmill-labs/windmill/issues/8508)) ([85c52e2](https://github.com/windmill-labs/windmill/commit/85c52e2cded10606cc895d0d3b717e13c69bc9b3))
+* use resource-level scope overrides during OAuth2 token refresh ([#8540](https://github.com/windmill-labs/windmill/issues/8540)) ([55ad0ff](https://github.com/windmill-labs/windmill/commit/55ad0ff5c499c33b766f47c6f32ba5d3eeb14763))
+
 ## [1.664.0](https://github.com/windmill-labs/windmill/compare/v1.663.0...v1.664.0) (2026-03-24)
 
 

backend/.sqlx/query-08522e494e34f4ecae21460262bf0ed3c5a197dd744c87cb760aaf47001febbd.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM trashbin WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "08522e494e34f4ecae21460262bf0ed3c5a197dd744c87cb760aaf47001febbd"
+}

backend/.sqlx/query-115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2",
+  "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -57,6 +57,11 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "disabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -76,8 +81,9 @@
       true,
       null,
       false,
+      false,
       false
     ]
   },
-  "hash": "05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab"
+  "hash": "115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc"
 }

backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d"
+}

backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM token WHERE email = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6"
+}

backend/.sqlx/query-1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
+  "query": "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
   "describe": {
     "columns": [
       {
@@ -50,11 +50,16 @@
       },
       {
         "ordinal": 9,
-        "name": "super_admin",
+        "name": "is_service_account",
         "type_info": "Bool"
       },
       {
         "ordinal": 10,
+        "name": "super_admin!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
         "name": "name",
         "type_info": "Varchar"
       }
@@ -76,8 +81,9 @@
       true,
       true,
       false,
+      null,
       true
     ]
   },
-  "hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
+  "hash": "1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1"
 }

backend/.sqlx/query-1d995dd5a094631ae96c16d68026fdeb22714af38162e87c02b052a5b8ec2645.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM trashbin WHERE workspace_id = $1 AND id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1d995dd5a094631ae96c16d68026fdeb22714af38162e87c02b052a5b8ec2645"
+}

backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            schemaname || '.' || relname as \"table_name!\",\n            pg_total_relation_size(relid) as \"total_size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n        FROM pg_catalog.pg_statio_user_tables\n        ORDER BY pg_total_relation_size(relid) DESC\n        LIMIT 15",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "total_size_bytes!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "total_size_pretty!",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6"
+}

backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, disabled FROM password WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "disabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443"
+}

backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2"
+}

backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            ws.workspace_id as \"workspace_id!\",\n            dt.key as \"name!\",\n            dt.value->>'table_name' as \"table_name\"\n        FROM workspace_settings ws,\n             jsonb_each(ws.datatable) dt\n        WHERE dt.value->>'resource_type' = 'instance'\n          AND dt.value->>'table_name' IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "table_name",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null,
+      null
+    ]
+  },
+  "hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720"
+}

backend/.sqlx/query-3453c0b7dd3c4d2c9bc639f379901741955502c9345e82a9b7fbbf3d3c7ab517.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM trashbin WHERE expires_at <= now()",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "3453c0b7dd3c4d2c9bc639f379901741955502c9345e82a9b7fbbf3d3c7ab517"
+}

backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "size_bytes",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "size_pretty",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null
+    ]
+  },
+  "hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0"
+}

backend/.sqlx/query-446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7.json

@@ -0,0 +1,65 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, workspace_id, item_kind, item_path, item_data, deleted_by, deleted_at, expires_at\n         FROM trashbin\n         WHERE workspace_id = $1 AND id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "item_kind",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "item_path",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "item_data",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 5,
+        "name": "deleted_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "deleted_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 7,
+        "name": "expires_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "446404eda9b9632c9a1384af6bf2f88594825dbaa647290a58bd63df61b531a7"
+}

backend/.sqlx/query-51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef.json

@@ -0,0 +1,61 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, workspace_id, item_kind, item_path, deleted_by, deleted_at, expires_at\n             FROM trashbin\n             WHERE workspace_id = $1 AND item_kind = $2\n             ORDER BY deleted_at DESC\n             LIMIT $3 OFFSET $4",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "item_kind",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "item_path",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "deleted_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "deleted_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 6,
+        "name": "expires_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "51c3274a8092d80503a6b97ef3896cc3ba1957042a48ac5f9629ada25b3e78ef"
+}

backend/.sqlx/query-52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT DISTINCT imported_path as \"imported_path!\"\n            FROM dependency_map\n            WHERE workspace_id = $1\n                AND importer_path = $2\n                AND imported_path NOT LIKE 'dependencies/%'\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "imported_path!",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "52d765c87cb8da0ca71fb53156820e383a998a54c95355bb85fe7e762a0d9765"
+}

backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, is_service_account, disabled FROM usr WHERE username = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_service_account",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "disabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb"
+}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -63,7 +68,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c"

backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -64,7 +69,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b"

backend/.sqlx/query-63c48fde8c0c0fff9abffc3be27e9948556b636b70b818cc31c2d50921a27366.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url FROM account WHERE workspace_id = $1 AND id = $2",
+  "query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, scopes FROM account WHERE workspace_id = $1 AND id = $2",
   "describe": {
     "columns": [
       {
@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "cc_token_url",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "scopes",
+        "type_info": "TextArray"
       }
     ],
     "parameters": {
@@ -46,8 +51,9 @@
       false,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "cc269052ffc1e613d7edc31f0f7bb84f6e6301ad1afb028813105a121a69fa7e"
+  "hash": "63c48fde8c0c0fff9abffc3be27e9948556b636b70b818cc31c2d50921a27366"
 }

backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND (username = $2 OR email = $3))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040"
+}

backend/.sqlx/query-6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_advisory_xact_lock(hashtext($1 || '/' || $2))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "pg_advisory_xact_lock",
+        "type_info": "Void"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6d070f476538aa6fcd6227fe5312561a7d098f2af5287e1e6c339e15080378be"
+}

backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            c.relname as \"table_name!\",\n            pg_total_relation_size(c.oid) as \"size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n            COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n        FROM pg_class c\n        JOIN pg_namespace n ON n.oid = c.relnamespace\n        WHERE n.nspname = 'public' AND c.relname = ANY($1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Name"
+      },
+      {
+        "ordinal": 1,
+        "name": "size_bytes!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "size_pretty!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "estimated_rows!",
+        "type_info": "Float4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "NameArray"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84"
+}

backend/.sqlx/query-870e1c3f0dc1aaa07ac74a2e37721ce352ad4fb67d36c19dce09d841e36f85dd.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO account (workspace_id, client, expires_at, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url) VALUES ($1, $2, now() + ($3 || ' seconds')::interval, $4, $5, $6, $7, $8, $9) RETURNING id",
+  "query": "INSERT INTO account (workspace_id, client, expires_at, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url, scopes) VALUES ($1, $2, now() + ($3 || ' seconds')::interval, $4, $5, $6, $7, $8, $9, $10) RETURNING id",
   "describe": {
     "columns": [
       {
@@ -19,12 +19,13 @@
         "Varchar",
         "Varchar",
         "Varchar",
-        "Text"
+        "Text",
+        "TextArray"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "b1bd088c2e1aca3104bede7d0953369b6b17ad3ad62692ae6f2303be890e6391"
+  "hash": "870e1c3f0dc1aaa07ac74a2e37721ce352ad4fb67d36c19dce09d841e36f85dd"
 }

backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "cnt",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9"
+}

backend/.sqlx/query-87ee10d8ba5ba281781f23e5390190fb90df980a19c900452f9b1a19c3620e30.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                        SELECT\n                            route_path,\n                            workspace_id,\n                            http_method::TEXT AS \"http_method!\"\n                        FROM\n                            http_trigger\n                        WHERE\n                            workspaced_route IS FALSE\n                            AND route_path_key IN (\n                                SELECT\n                                    route_path_key\n                                FROM\n                                    http_trigger\n                                WHERE\n                                    workspaced_route IS FALSE\n                                GROUP BY\n                                    route_path_key, http_method\n                                HAVING COUNT(*) > 1\n                            )\n                        ORDER BY route_path_key\n                    ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "http_method!",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "87ee10d8ba5ba281781f23e5390190fb90df980a19c900452f9b1a19c3620e30"
+}

backend/.sqlx/query-8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO trashbin (workspace_id, item_kind, item_path, item_data, deleted_by)\n         VALUES ($1, $2, $3, $4, $5) RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Text",
+        "Jsonb",
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "8b25c4252da77cd2fe1b3916b518251dbb3c6d4c095efa015823f0324ab27d7f"
+}

backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE password SET disabled = $1 WHERE email = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67"
+}

backend/.sqlx/query-92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32.json

@@ -0,0 +1,60 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, workspace_id, item_kind, item_path, deleted_by, deleted_at, expires_at\n             FROM trashbin\n             WHERE workspace_id = $1\n             ORDER BY deleted_at DESC\n             LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "item_kind",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "item_path",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "deleted_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "deleted_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 6,
+        "name": "expires_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "92fb6afe3b7041b2954340094c08e702fc1577d3fa4ff1ff2f1e089971ff5e32"
+}

backend/.sqlx/query-97cf826b271cf064182382c924188fee392ed9cff6ae446abc86170984304a25.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT args as \"args: sqlx::types::Json<Box<RawValue>>\"\n                        FROM v2_job\n                        WHERE id = $1",
+  "query": "SELECT args as \"args: sqlx::types::Json<Box<RawValue>>\"\n                    FROM v2_job\n                    WHERE id = $1",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       true
     ]
   },
-  "hash": "d1dcc7fc8a1e1bc4dad263ec5163a94fca9dd95cc3b26b33611eab9d2a261141"
+  "hash": "97cf826b271cf064182382c924188fee392ed9cff6ae446abc86170984304a25"
 }

backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
   "describe": {
     "columns": [
       {
@@ -12,6 +12,11 @@
         "ordinal": 1,
         "name": "deployment_msg",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamptz"
       }
     ],
     "parameters": {
@@ -22,8 +27,9 @@
     },
     "nullable": [
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec"
+  "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c"
 }

backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM usr WHERE is_service_account = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf"
+}

backend/.sqlx/query-a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n            authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n            SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source\n            FROM password\n            WHERE email IN (SELECT email FROM active_users)\n            ORDER BY super_admin DESC, devops DESC\n            LIMIT $1 OFFSET $2",
+  "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n            authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n            SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled\n            FROM password\n            WHERE email IN (SELECT email FROM active_users)\n            ORDER BY super_admin DESC, devops DESC\n            LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -57,6 +57,11 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "disabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -76,8 +81,9 @@
       true,
       true,
       false,
+      false,
       false
     ]
   },
-  "hash": "60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce"
+  "hash": "a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10"
 }

backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT AVG(pg_column_size(result))::bigint as \"avg_size\"\n        FROM (\n            SELECT result FROM v2_job_completed\n            WHERE completed_at > now() - interval '30 days'\n              AND result IS NOT NULL\n            ORDER BY completed_at DESC\n            LIMIT $1\n        ) sub",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "avg_size",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00"
+}

backend/.sqlx/query-add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43"
+}

backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "oldest",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 1,
+        "name": "total",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null
+    ]
+  },
+  "hash": "b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30"
+}

backend/.sqlx/query-bae31609123da68d16bea8e0f1c4624403b6f97e13f13f056501fe2f4efb0f06.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM trashbin WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "bae31609123da68d16bea8e0f1c4624403b6f97e13f13f056501fe2f4efb0f06"
+}

backend/.sqlx/query-bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3.json

@@ -0,0 +1,44 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            schemaname || '.' || relname as \"table_name!\",\n            COALESCE(n_live_tup, 0) as \"live_tuples!\",\n            COALESCE(n_dead_tup, 0) as \"dead_tuples!\",\n            last_autovacuum as \"last_autovacuum\",\n            last_autoanalyze as \"last_autoanalyze\"\n        FROM pg_stat_user_tables\n        ORDER BY n_dead_tup DESC\n        LIMIT 15",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "live_tuples!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "dead_tuples!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 3,
+        "name": "last_autovacuum",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "last_autoanalyze",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      true,
+      true
+    ]
+  },
+  "hash": "bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3"
+}

backend/.sqlx/query-c1fd495abb4353b46361ec94fd4ae8d224457171b1b73fe145d28e67f1fe03af.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "c1fd495abb4353b46361ec94fd4ae8d224457171b1b73fe145d28e67f1fe03af"
+}

backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
   "describe": {
     "columns": [
       {
@@ -12,6 +12,11 @@
         "ordinal": 1,
         "name": "deployment_msg",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamptz"
       }
     ],
     "parameters": {
@@ -22,8 +27,9 @@
     },
     "nullable": [
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de"
+  "hash": "c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462"
 }

backend/.sqlx/query-ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT super_admin FROM password WHERE email = $1 AND disabled = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "super_admin",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21"
+}

backend/.sqlx/query-daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT DO NOTHING",
+  "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT (email) DO UPDATE SET disabled = false",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6"
+  "hash": "daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656"
 }

backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json

@@ -0,0 +1,46 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            c.id as \"id!\",\n            c.workspace_id as \"workspace_id!\",\n            j.runnable_path as \"runnable_path\",\n            pg_column_size(c.result) as \"result_size_bytes!\",\n            c.completed_at as \"completed_at!\"\n        FROM (\n            SELECT id, workspace_id, result, completed_at\n            FROM v2_job_completed\n            WHERE completed_at > now() - interval '30 days'\n              AND result IS NOT NULL\n            ORDER BY completed_at DESC\n            LIMIT $1\n        ) c\n        LEFT JOIN v2_job j ON j.id = c.id\n        WHERE pg_column_size(c.result) > 1024\n        ORDER BY pg_column_size(c.result) DESC\n        LIMIT 10",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "runnable_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "result_size_bytes!",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "completed_at!",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      null,
+      false
+    ]
+  },
+  "hash": "dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7"
+}

backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -63,7 +68,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2"

backend/.sqlx/query-f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password WHERE email = $1",
+  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE email = $1",
   "describe": {
     "columns": [
       {
@@ -57,6 +57,11 @@
         "ordinal": 10,
         "name": "role_source",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "disabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -75,8 +80,9 @@
       true,
       null,
       false,
+      false,
       false
     ]
   },
-  "hash": "65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9"
+  "hash": "f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493"
 }

backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json

@@ -0,0 +1,104 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT DISTINCT ON (path) path, language AS \"language: ScriptLang\", content FROM script\n         WHERE workspace_id = $1\n           AND archived = false\n           AND dedicated_worker = true\n           AND language = ANY($2::SCRIPT_LANG[])\n         ORDER BY path, created_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp",
+                "oracledb",
+                "nu",
+                "java",
+                "duckdb",
+                "ruby"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "script_lang[]",
+            "kind": {
+              "Array": {
+                "Custom": {
+                  "name": "script_lang",
+                  "kind": {
+                    "Enum": [
+                      "python3",
+                      "deno",
+                      "go",
+                      "bash",
+                      "postgresql",
+                      "nativets",
+                      "bun",
+                      "mysql",
+                      "bigquery",
+                      "snowflake",
+                      "graphql",
+                      "powershell",
+                      "mssql",
+                      "php",
+                      "bunnative",
+                      "rust",
+                      "ansible",
+                      "csharp",
+                      "oracledb",
+                      "nu",
+                      "java",
+                      "duckdb",
+                      "ruby"
+                    ]
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6"
+}

backend/.sqlx/query-f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token\n            (token_hash, token_prefix, token, email, label, expiration, super_admin, owner)\n            VALUES ($1, $2, $3, $4, $5, $6, false, $7)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b"
+}

backend/.sqlx/query-f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr\n            (workspace_id, email, username, is_admin, operator, is_service_account)\n            VALUES ($1, $2, $3, false, true, true)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853"
+}

backend/.sqlx/query-fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT disabled FROM password WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "disabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924"
+}

backend/.sqlx/query-fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO token\n                    (token_hash, token_prefix, token, label, super_admin, email)\n                    VALUES ($1, $2, $3, $4, $5, $6)",
+  "query": "INSERT INTO token\n            (token_hash, token_prefix, token, label, super_admin, email)\n            VALUES ($1, $2, $3, $4, $5, $6)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -15,5 +15,5 @@
     },
     "nullable": []
   },
-  "hash": "d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90"
+  "hash": "fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.664.0"
+version = "1.668.2"
 authors.workspace = true
 edition.workspace = true
 
@@ -82,7 +82,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.664.0"
+version = "1.668.2"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -260,6 +260,8 @@ windmill-dep-map.workspace = true
 windmill-test-utils.workspace = true
 windmill-worker-volumes.workspace = true
 windmill-types.workspace = true
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 windmill-trigger.workspace = true
 windmill-trigger-websocket.workspace = true
 windmill-trigger-postgres.workspace = true
@@ -362,7 +364,7 @@ reqwest-middleware = { version = "^0", features = ["json"] }
 
 bitflags = "2.9.4"
 memchr = "2.7.4"
-axum = { version = "^0.7", features = ["multipart", "macros"] }
+axum = { version = "^0.8", features = ["multipart", "macros"] }
 headers = "^0"
 hyper = { version = "^1", features = ["full"] }
 hyper-tls = "^0.6"
@@ -371,7 +373,7 @@ tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
 tokio-stream = { version = "0.1.17" }
 tower = "^0"
 tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
-tower-cookies = "^0.10"
+tower-cookies = "^0.11"
 #stuck because of swc for now
 serde = "=1.0.220"
 serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
@@ -386,7 +388,7 @@ tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 tracing-appender = "^0"
 prometheus = { version = "^0", default-features = false }
-cookie = { version = "0.17.0" }
+cookie = { version = "0.18.0" }
 phf = { version = "0.11", features = ["macros"] }
 rust-embed = { version = "^6", features = ["interpolate-folder-path"] }
 mime_guess = "^2"
@@ -415,6 +417,7 @@ time = "^0"
 serde_urlencoded = "^0"
 astral-tokio-tar = "^0.5.6"
 tempfile = "^3"
+x509-parser = "^0.16"
 tokio-util = { version = "=0.7.17", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0.14.0"
@@ -566,18 +569,18 @@ flate2 = "^1"
 http = "^1"
 async-stream = "^0"
 
-opentelemetry = "0.27.0"
-tracing-opentelemetry = "0.28.0"
-opentelemetry_sdk = { version = "0.27.1", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
-opentelemetry-appender-tracing = "0.27.0"
-opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
-opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] }
+opentelemetry = "0.30.0"
+tracing-opentelemetry = "0.31.0"
+opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] }
+opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls"] }
+opentelemetry-appender-tracing = "0.30.0"
+opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] }
+opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] }
 prost = "0.13"
 
 bollard = "0.18.1"
 
-tonic = { version = "=0.12.3", features = ["tls-native-roots"] }
+tonic = { version = "^0.13", features = ["tls-native-roots"] }
 byteorder = "1.5.0"
 
 tikv-jemallocator = { version = "0.5" }

backend/ee-repo-ref.txt

@@ -1 +1 @@
-b5c8af4df9ba2c39fdd494d7a40f9a92fbff8abc
+02c0d34e54e71c9293f9cefb56f68652cf0db8a5

backend/migrations/20260324000000_scim_deactivated_user.down.sql

@@ -0,0 +1 @@
+ALTER TABLE password DROP COLUMN IF EXISTS disabled;

backend/migrations/20260324000000_scim_deactivated_user.up.sql

@@ -0,0 +1 @@
+ALTER TABLE password ADD COLUMN disabled BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20260325000000_app_bundles_grants.down.sql

@@ -0,0 +1,3 @@
+-- Revoke grants for app_bundles table
+REVOKE ALL ON app_bundles FROM windmill_user;
+REVOKE ALL ON app_bundles FROM windmill_admin;

backend/migrations/20260325000000_app_bundles_grants.up.sql

@@ -0,0 +1,3 @@
+-- Add grants for app_bundles table
+GRANT ALL ON app_bundles TO windmill_user;
+GRANT ALL ON app_bundles TO windmill_admin;

backend/migrations/20260326000000_trashbin.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS trashbin;

backend/migrations/20260326000000_trashbin.up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE trashbin (
+    id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
+    item_kind VARCHAR(50) NOT NULL,
+    item_path TEXT NOT NULL,
+    item_data JSONB NOT NULL,
+    deleted_by VARCHAR(255) NOT NULL,
+    deleted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    expires_at TIMESTAMPTZ NOT NULL DEFAULT now() + INTERVAL '3 days'
+);
+
+CREATE INDEX idx_trashbin_expires_at ON trashbin(expires_at);
+CREATE INDEX idx_trashbin_workspace_kind ON trashbin(workspace_id, item_kind);
+
+GRANT ALL ON trashbin TO windmill_user;
+GRANT ALL ON trashbin TO windmill_admin;

backend/migrations/20260326100000_account_scopes.down.sql

@@ -0,0 +1 @@
+ALTER TABLE account DROP COLUMN IF EXISTS scopes;

backend/migrations/20260326100000_account_scopes.up.sql

@@ -0,0 +1 @@
+ALTER TABLE account ADD COLUMN scopes TEXT[];

backend/migrations/20260326200000_service_accounts.down.sql

@@ -0,0 +1 @@
+ALTER TABLE usr DROP COLUMN is_service_account;

backend/migrations/20260326200000_service_accounts.up.sql

@@ -0,0 +1 @@
+ALTER TABLE usr ADD COLUMN IF NOT EXISTS is_service_account BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20260327000000_email_varchar_255.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(50);
+ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(50);

backend/migrations/20260327000000_email_varchar_255.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(255);

backend/src/main.rs

@@ -44,17 +44,18 @@ use windmill_common::{
         CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
-        INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
+        HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING,
+        INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
         JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
         KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING,
         MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
         NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
         OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
         POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
-        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
-        TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING,
+        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
+        RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
+        SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
+        UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING,
     },
     scripts::ScriptLang,
     stats_oss::schedule_stats,
@@ -67,7 +68,7 @@ use windmill_common::{
         is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
         HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
     },
-    KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
+    KillpillSender, DEFAULT_HUB_BASE_URL, INSTANCE_NAME, METRICS_ENABLED,
 };
 
 #[cfg(feature = "enterprise")]
@@ -104,10 +105,10 @@ use crate::monitor::{
     reload_base_url_setting, reload_bunfig_install_scopes_setting,
     reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting,
     reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
-    reload_hub_api_secret_setting, reload_hub_base_url_setting,
-    reload_instance_events_webhook_setting, reload_job_default_timeout_setting,
-    reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
-    reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
+    reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting,
+    reload_hub_base_url_setting, reload_instance_events_webhook_setting,
+    reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
+    reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
     reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
     reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
 };
@@ -1099,6 +1100,9 @@ Windmill Community Edition {GIT_VERSION}
         }
 
         let addr = SocketAddr::from((server_bind_address, port));
+        let listener = tokio::net::TcpListener::bind(addr)
+            .await
+            .context("binding main windmill server")?;
 
         let (base_internal_tx, base_internal_rx) = tokio::sync::oneshot::channel::<String>();
 
@@ -1232,7 +1236,7 @@ Windmill Community Edition {GIT_VERSION}
                         db.clone(),
                         index_reader,
                         log_index_reader,
-                        addr,
+                        listener,
                         server_killpill_rx,
                         base_internal_tx,
                         server_mode,
@@ -1788,7 +1792,8 @@ async fn process_notify_event(
                     reload_otel_tracing_proxy_setting(conn).await;
                     if worker_mode {
                         tracing::info!("OTEL tracing proxy setting changed, restarting worker");
-                        send_delayed_killpill(tx, 4, "OTEL tracing proxy setting change").await;
+                        spawn_graceful_killpill(tx, db, 10, "OTEL tracing proxy setting change")
+                            .await;
                     }
                 }
                 REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
@@ -1796,12 +1801,12 @@ async fn process_notify_event(
                 }
                 EXPOSE_METRICS_SETTING => {
                     tracing::info!("Metrics setting changed, restarting");
-                    send_delayed_killpill(tx, 40, "metrics setting change").await;
+                    spawn_graceful_killpill(tx, db, 10, "metrics setting change").await;
                 }
                 EMAIL_DOMAIN_SETTING => {
                     tracing::info!("Email domain setting changed");
                     if server_mode {
-                        send_delayed_killpill(tx, 4, "email domain setting change").await;
+                        spawn_graceful_killpill(tx, db, 10, "email domain setting change").await;
                     }
                 }
                 EXPOSE_DEBUG_METRICS_SETTING => {
@@ -1814,25 +1819,42 @@ async fn process_notify_event(
                         tracing::error!(error = %e, "Could not reload app workspaced route setting");
                     }
                 }
+                HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => {
+                    if let Err(e) = reload_http_route_workspaced_route_setting(db).await {
+                        tracing::error!(error = %e, "Could not reload http route workspaced route setting");
+                    }
+                    #[cfg(feature = "http_trigger")]
+                    match windmill_api::triggers::http::refresh_routers(db).await {
+                        Ok((true, _)) => {
+                            tracing::info!(
+                                "Refreshed HTTP routers (http workspaced route setting change)"
+                            );
+                        }
+                        Err(err) => {
+                            tracing::error!("Error refreshing HTTP routers (http workspaced route setting change): {err:#}");
+                        }
+                        _ => {}
+                    }
+                }
                 AI_CONFIG_SETTING => {
                     tracing::info!("AI config setting changed, bumping instance AI cache revision");
                     bump_instance_ai_config_revision();
                 }
                 OTEL_SETTING => {
                     tracing::info!("OTEL setting changed, restarting");
-                    send_delayed_killpill(tx, 4, "OTEL setting change").await;
+                    spawn_graceful_killpill(tx, db, 10, "OTEL setting change").await;
                 }
                 REQUEST_SIZE_LIMIT_SETTING => {
                     if server_mode {
                         tracing::info!("Request limit size change detected, killing server expecting to be restarted");
-                        send_delayed_killpill(tx, 4, "request size limit change").await;
+                        spawn_graceful_killpill(tx, db, 10, "request size limit change").await;
                     }
                 }
                 SAML_METADATA_SETTING => {
                     tracing::info!(
                         "SAML metadata change detected, killing server expecting to be restarted"
                     );
-                    send_delayed_killpill(tx, 0, "SAML metadata change").await;
+                    spawn_graceful_killpill(tx, db, 10, "SAML metadata change").await;
                 }
                 HUB_BASE_URL_SETTING => {
                     if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
@@ -1881,6 +1903,9 @@ async fn process_notify_event(
                     .unwrap_or(false);
                     tracing::info!("Workspace telemetry setting changed: enabled={}", enabled);
                 }
+                RESTART_COORDINATION_SETTING => {
+                    // Internal coordination key for staggered restarts, no action needed
+                }
                 _ => {
                     tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload);
                 }
@@ -2022,14 +2047,145 @@ pub async fn run_workers(
     Ok(())
 }
 
-async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
-    if max_delay_secs == 0 {
-        max_delay_secs = 1;
-    }
-    // Random delay to avoid all servers/workers shutting down simultaneously
-    let rd_delay = rand::rng().random_range(0..max_delay_secs);
-    tracing::info!("Scheduling {context} shutdown in {rd_delay}s");
-    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+/// Schedule a graceful restart with DB-coordinated staggering.
+///
+/// Uses a PostgreSQL advisory lock to serialize restart scheduling across server instances.
+/// Each instance records its planned restart time in the `_restart_coordination` global setting;
+/// subsequent instances read existing schedules and shift their restart to maintain at least
+/// `safety_margin_secs` between consecutive restarts (must exceed the server startup time).
+///
+/// Every server waits at least `DRAIN_DELAY_SECS` to let in-flight requests complete.
+/// Each subsequent server waits an additional `safety_margin_secs` after the previous one,
+/// guaranteeing zero downtime overlap.
+///
+/// The DB coordination is done synchronously (fast, ~ms) to reserve our restart slot,
+/// then the sleep+kill is spawned in the background so the notification handler is not blocked.
+///
+/// Falls back to drain-only delay if DB coordination fails.
+async fn spawn_graceful_killpill(
+    tx: &KillpillSender,
+    db: &Pool<Postgres>,
+    safety_margin_secs: u64,
+    context: &str,
+) {
+    // Minimum delay before any restart to let in-flight requests drain
+    const DRAIN_DELAY_SECS: u64 = 3;
 
-    tx.send();
+    let delay = match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await {
+        Ok(d) => d,
+        Err(e) => {
+            tracing::warn!(
+                "Failed to coordinate restart for {context}: {e:#}, \
+                 falling back to drain delay of {DRAIN_DELAY_SECS}s"
+            );
+            DRAIN_DELAY_SECS
+        }
+    };
+
+    tracing::info!("Scheduling {context} graceful shutdown in {delay}s");
+    let tx = tx.clone();
+    tokio::spawn(async move {
+        tokio::time::sleep(Duration::from_secs(delay)).await;
+        tx.send();
+    });
+}
+
+/// Coordinate a restart delay with other instances via the DB.
+///
+/// Returns the delay (in seconds from now) at which this instance should restart.
+/// The first server gets `drain_delay_secs` (to let in-flight requests complete).
+/// Each subsequent server is spaced `safety_margin_secs` after the latest scheduled restart.
+async fn coordinate_restart_delay(
+    db: &Pool<Postgres>,
+    safety_margin_secs: u64,
+    drain_delay_secs: u64,
+) -> anyhow::Result<u64> {
+    const RESTART_LOCK_ID: i64 = 737_483_920;
+    // Stale threshold: ignore coordination entries older than this
+    const STALE_THRESHOLD_SECS: i64 = 120;
+
+    let now = chrono::Utc::now();
+
+    let mut tx = db.begin().await.context("begin restart coordination tx")?;
+
+    // Serialize access across all instances
+    sqlx::query("SELECT pg_advisory_xact_lock($1)")
+        .bind(RESTART_LOCK_ID)
+        .execute(&mut *tx)
+        .await
+        .context("acquire restart coordination lock")?;
+
+    // Read existing coordination record
+    let existing: Option<serde_json::Value> =
+        sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1")
+            .bind(RESTART_COORDINATION_SETTING)
+            .fetch_optional(&mut *tx)
+            .await
+            .context("read restart coordination")?;
+
+    // Parse existing scheduled restarts, filtering out stale entries
+    // Each entry is (instance_name, restart_at)
+    let mut scheduled: Vec<(String, chrono::DateTime<chrono::Utc>)> = Vec::new();
+    if let Some(val) = &existing {
+        if let Some(arr) = val.get("restarts").and_then(|v| v.as_array()) {
+            for entry in arr {
+                let instance = entry
+                    .get("instance")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("unknown")
+                    .to_string();
+                if let Some(ts_str) = entry.get("restart_at").and_then(|v| v.as_str()) {
+                    if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(ts_str) {
+                        let dt = dt.with_timezone(&chrono::Utc);
+                        let stale_cutoff = now - chrono::Duration::seconds(STALE_THRESHOLD_SECS);
+                        if dt > stale_cutoff {
+                            scheduled.push((instance, dt));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Find the latest scheduled restart
+    let latest = scheduled.iter().map(|(_, dt)| *dt).max();
+    let earliest_allowed = now + chrono::Duration::seconds(drain_delay_secs as i64);
+
+    // Our restart time: drain_delay from now, or safety_margin after the latest existing restart
+    let our_restart = match latest {
+        Some(last) => {
+            let after_last = last + chrono::Duration::seconds(safety_margin_secs as i64);
+            // Use whichever is later: drain delay or staggered position
+            earliest_allowed.max(after_last)
+        }
+        None => earliest_allowed,
+    };
+
+    // Record our restart time (deduplicate: remove any prior entry for this instance)
+    scheduled.retain(|(inst, _)| inst != &*INSTANCE_NAME);
+    scheduled.push((INSTANCE_NAME.clone(), our_restart));
+    let new_value = serde_json::json!({
+        "restarts": scheduled.iter().map(|(inst, dt)| {
+            serde_json::json!({
+                "instance": inst,
+                "restart_at": dt.to_rfc3339()
+            })
+        }).collect::<Vec<_>>()
+    });
+
+    sqlx::query(
+        "INSERT INTO global_settings (name, value, updated_at) \
+         VALUES ($1, $2, now()) \
+         ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+    )
+    .bind(RESTART_COORDINATION_SETTING)
+    .bind(&new_value)
+    .execute(&mut *tx)
+    .await
+    .context("write restart coordination")?;
+
+    tx.commit().await.context("commit restart coordination")?;
+
+    let delay = (our_restart - now).num_seconds().max(0) as u64;
+    Ok(delay)
 }

backend/src/monitor.rs

@@ -88,7 +88,13 @@ use windmill_common::{
     MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
     SERVICE_LOG_RETENTION_SECS,
 };
-use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING};
+use windmill_common::{
+    client::AuthedClient,
+    global_settings::{
+        APP_WORKSPACED_ROUTE_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE,
+        HTTP_ROUTE_WORKSPACED_ROUTE_SETTING,
+    },
+};
 #[cfg(feature = "parquet")]
 use windmill_object_store::reload_object_store_setting;
 use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
@@ -163,6 +169,8 @@ lazy_static::lazy_static! {
 
     static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref QUEUE_RUNNING_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+    static ref OTEL_QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+    static ref OTEL_QUEUE_RUNNING_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
 
     //legacy typo
@@ -296,6 +304,10 @@ pub async fn initial_load(
         if let Err(e) = reload_app_workspaced_route_setting(db).await {
             tracing::error!("Error reloading app workspaced route: {:?}", e)
         }
+
+        if let Err(e) = reload_http_route_workspaced_route_setting(db).await {
+            tracing::error!("Error reloading http route workspaced route: {:?}", e)
+        }
     }
 
     #[cfg(feature = "parquet")]
@@ -1168,6 +1180,15 @@ pub async fn delete_expired_items(db: &DB) -> () {
             tracing::error!("Error deleting custom concurrency key: {:?}", e);
         }
     }
+
+    match windmill_common::trashbin::delete_expired_trash(db).await {
+        Ok(count) => {
+            if count > 0 {
+                tracing::info!("deleted {} expired trash items", count);
+            }
+        }
+        Err(e) => tracing::error!("Error deleting expired trash items: {}", e.to_string()),
+    }
 }
 
 pub async fn check_expiring_tokens(db: &DB) {
@@ -2353,8 +2374,20 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             }
         }
 
+        let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed);
+
+        if otel_enabled {
+            for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() {
+                if queue_counts.get(q).is_none() {
+                    otel_set_queue_count(q, 0);
+                }
+            }
+        }
+
         #[allow(unused_mut)]
         let mut tags_to_watch = vec![];
+        #[allow(unused_mut)]
+        let mut otel_tags_to_watch = vec![];
         for q in queue_counts {
             let count = q.1;
             let tag = q.0;
@@ -2366,6 +2399,9 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 tags_to_watch.push(tag.to_string());
             }
 
+            if otel_enabled {
+                otel_tags_to_watch.push(tag.to_string());
+            }
             otel_set_queue_count(&tag, count as i64);
 
             // save queue_count and delay metrics per tag
@@ -2400,9 +2436,13 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             let mut w = QUEUE_COUNT_TAGS.write().await;
             *w = tags_to_watch;
         }
+        if otel_enabled {
+            let mut w = OTEL_QUEUE_COUNT_TAGS.write().await;
+            *w = otel_tags_to_watch;
+        }
 
         // Single DB query for running counts, shared by Prometheus and OTel
-        let otel_running = OTEL_METRICS_ENABLED.load(Ordering::Relaxed);
+        let otel_running = otel_enabled;
         #[cfg(feature = "prometheus")]
         let need_running_counts = metrics_enabled || otel_running;
         #[cfg(not(feature = "prometheus"))]
@@ -2420,8 +2460,18 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 }
             }
 
+            if otel_running {
+                for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() {
+                    if queue_running_counts.get(q).is_none() {
+                        otel_set_queue_running_count(q, 0);
+                    }
+                }
+            }
+
             #[allow(unused_mut, unused_variables)]
             let mut running_tags_to_watch: Vec<String> = vec![];
+            #[allow(unused_mut, unused_variables)]
+            let mut otel_running_tags_to_watch: Vec<String> = vec![];
             for (tag, count) in &queue_running_counts {
                 #[cfg(feature = "prometheus")]
                 if metrics_enabled {
@@ -2432,6 +2482,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
 
                 if otel_running {
                     otel_set_queue_running_count(tag, *count as i64);
+                    otel_running_tags_to_watch.push(tag.to_string());
                 }
             }
 
@@ -2440,6 +2491,10 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await;
                 *w = running_tags_to_watch;
             }
+            if otel_running {
+                let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await;
+                *w = otel_running_tags_to_watch;
+            }
         }
     }
 
@@ -3390,6 +3445,39 @@ pub async fn reload_app_workspaced_route_setting(conn: &DB) -> error::Result<()>
     Ok(())
 }
 
+pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> {
+    let http_route_workspaced_route =
+        load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?;
+
+    let ws_route = match http_route_workspaced_route {
+        Some(serde_json::Value::Bool(ws_route)) => ws_route,
+        None => false,
+        _ => {
+            tracing::error!(
+                "Expected {} to be a boolean got: {:?}. Defaulting to false",
+                HTTP_ROUTE_WORKSPACED_ROUTE_SETTING,
+                http_route_workspaced_route
+            );
+            false
+        }
+    };
+
+    let mut l = HTTP_ROUTE_WORKSPACED_ROUTE.write().await;
+
+    if *l != ws_route {
+        *l = ws_route;
+        drop(l);
+        // Bump the HTTP trigger version so the route cache is rebuilt with
+        // the updated workspaced_route behavior on the next request.
+        sqlx::query!("SELECT nextval('http_trigger_version_seq')")
+            .fetch_one(conn)
+            .await?;
+    } else {
+        *l = ws_route;
+    }
+    Ok(())
+}
+
 pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<()> {
     #[derive(Deserialize)]
     struct DBOversize {

backend/summarized_schema.txt

@@ -33,7 +33,7 @@ workspace_key_kind: cloud
 
 ## Tables
 _sqlx_migrations: version(bigint), description(text), installed_on(ts), success(bool), checksum(bytes), execution_time(bigint)
-account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), client(char), refresh_error(text), grant_type(char), cc_client_id(char), cc_client_secret(char), cc_token_url(char), mcp_server_url(text)
+account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), client(char), refresh_error(text), grant_type(char), cc_client_id(char), cc_client_secret(char), cc_token_url(char), mcp_server_url(text), scopes(text[])
   FK: (workspace_id) -> workspace(id)
 agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char)
 ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts)
@@ -151,6 +151,9 @@ script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), s
 skip_workspace_diff_tally: workspace_id(char), added_at(ts)
 sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attributes(text[]), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error(text), server_id(char), last_server_ping(ts), aws_auth_resource_type(aws_auth_resource_type), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode)
   FK: (workspace_id) -> workspace(id)
+trashbin: id(bigint), workspace_id(char), item_kind(char), item_path(char), item_data(jsonb), deleted_by(char), deleted_at(ts), expires_at(ts)
+  FK: (workspace_id) -> workspace(id)
+  INDEX: idx_trashbin_expires_at (expires_at), idx_trashbin_workspace_kind (workspace_id, item_kind)
 token: token_hash(char), token_prefix(char), token(char), label(char), expiration(ts), workspace_id(char), owner(char), email(char), super_admin(bool), created_at(ts), last_used_at(ts), scopes(text[]), job(uuid)
   FK: (workspace_id) -> workspace(id)
 token_expiry_notification: token_hash(char), expiration(ts)

backend/tests/bun_jobs.rs

@@ -891,25 +891,34 @@ mod dedicated_worker_protocol {
     use std::process::{Command, Stdio};
     use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
     use windmill_worker::{
-        build_loader, generate_dedicated_worker_wrapper, LoaderMode, BUN_DEDICATED_WORKER_ARGS,
-        BUN_PATH, NODE_BIN_PATH,
+        build_loader, compute_ts_codegen, generate_multi_script_wrapper, LoaderMode, TsScriptEntry,
+        BUN_DEDICATED_WORKER_ARGS, BUN_PATH, NODE_BIN_PATH,
     };
 
+    const TEST_SCRIPT_PATH: &str = "f/test/script";
+
     /// Creates test worker files and optionally bundles for Node.js (like production)
     /// Returns the path to the wrapper file to execute
     fn create_test_worker_files(
         dir: &std::path::Path,
         script: &str,
-        arg_names: &[&str],
         bundle_for_node: bool,
     ) -> std::path::PathBuf {
         let dir_str = dir.to_str().unwrap();
+        // Write main.ts at root (like production single-script)
         std::fs::write(dir.join("main.ts"), script).unwrap();
 
+        let codegen = compute_ts_codegen(script);
+        let ext = if bundle_for_node { "js" } else { "ts" };
+        let scripts = [TsScriptEntry {
+            import_name: "main",
+            original_path: TEST_SCRIPT_PATH,
+            codegen: &codegen,
+        }];
+        let wrapper = generate_multi_script_wrapper(&scripts, ext);
+
         if bundle_for_node {
-            // For Node.js: bundle to JavaScript first (like production's build_loader with LoaderMode::Node)
-            let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None, None);
-            std::fs::write(dir.join("wrapper.mjs"), wrapper).unwrap();
+            std::fs::write(dir.join("wrapper.mjs"), &wrapper).unwrap();
 
             // Use the exact same build_loader function as production
             tokio::runtime::Runtime::new()
@@ -919,7 +928,7 @@ mod dedicated_worker_protocol {
                     "http://localhost:8000",
                     "test_token",
                     "test-workspace",
-                    "f/test/script",
+                    TEST_SCRIPT_PATH,
                     LoaderMode::Node,
                     &None,
                 ))
@@ -945,10 +954,8 @@ mod dedicated_worker_protocol {
             std::fs::rename(&bundled_path, &output_path).unwrap();
             output_path
         } else {
-            // For Bun: use TypeScript directly (like production)
-            let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None, None);
             let wrapper_path = dir.join("wrapper.mjs");
-            std::fs::write(&wrapper_path, wrapper).unwrap();
+            std::fs::write(&wrapper_path, &wrapper).unwrap();
             wrapper_path
         }
     }
@@ -957,14 +964,12 @@ mod dedicated_worker_protocol {
     fn run_worker_test(
         runtime: &str,
         script: &str,
-        arg_names: &[&str],
         jobs: Vec<serde_json::Value>,
     ) -> Vec<Result<serde_json::Value, String>> {
         let temp_dir = tempfile::tempdir().unwrap();
 
         // Create files and get the wrapper path (bundled for node, raw for bun)
-        let wrapper_path =
-            create_test_worker_files(temp_dir.path(), script, arg_names, runtime == "node");
+        let wrapper_path = create_test_worker_files(temp_dir.path(), script, runtime == "node");
         let wrapper_str = wrapper_path.to_str().unwrap();
 
         // Build args matching production behavior
@@ -1008,7 +1013,8 @@ mod dedicated_worker_protocol {
         let mut results = Vec::new();
 
         for job_args in jobs {
-            writeln!(stdin, "{}", job_args.to_string()).unwrap();
+            // Protocol: exec:<script_path>:<json_args>
+            writeln!(stdin, "exec:{}:{}", TEST_SCRIPT_PATH, job_args.to_string()).unwrap();
             stdin.flush().unwrap();
 
             let mut response = String::new();
@@ -1043,12 +1049,7 @@ export function main(x: number, y: number): number {
     return x + y;
 }
 "#;
-        let results = run_worker_test(
-            "node",
-            script,
-            &["x", "y"],
-            vec![serde_json::json!({"x": 5, "y": 3})],
-        );
+        let results = run_worker_test("node", script, vec![serde_json::json!({"x": 5, "y": 3})]);
 
         assert_eq!(results.len(), 1);
         assert_eq!(results[0], Ok(serde_json::json!(8)));
@@ -1062,7 +1063,7 @@ export function main(n: number): number {
 }
 "#;
         let jobs: Vec<serde_json::Value> = (1..=5).map(|i| serde_json::json!({"n": i})).collect();
-        let results = run_worker_test("node", script, &["n"], jobs);
+        let results = run_worker_test("node", script, jobs);
 
         assert_eq!(results.len(), 5);
         for (i, result) in results.iter().enumerate() {
@@ -1081,7 +1082,6 @@ export function main(msg: string): never {
         let results = run_worker_test(
             "node",
             script,
-            &["msg"],
             vec![serde_json::json!({"msg": "test error"})],
         );
 
@@ -1099,12 +1099,7 @@ export function main(x: number, y: number): number {
     return x + y;
 }
 "#;
-        let results = run_worker_test(
-            "bun",
-            script,
-            &["x", "y"],
-            vec![serde_json::json!({"x": 5, "y": 3})],
-        );
+        let results = run_worker_test("bun", script, vec![serde_json::json!({"x": 5, "y": 3})]);
 
         assert_eq!(results.len(), 1);
         assert_eq!(results[0], Ok(serde_json::json!(8)));
@@ -1118,7 +1113,7 @@ export function main(n: number): number {
 }
 "#;
         let jobs: Vec<serde_json::Value> = (1..=5).map(|i| serde_json::json!({"n": i})).collect();
-        let results = run_worker_test("bun", script, &["n"], jobs);
+        let results = run_worker_test("bun", script, jobs);
 
         assert_eq!(results.len(), 5);
         for (i, result) in results.iter().enumerate() {
@@ -1137,7 +1132,6 @@ export function main(msg: string): never {
         let results = run_worker_test(
             "bun",
             script,
-            &["msg"],
             vec![serde_json::json!({"msg": "test error"})],
         );
 
@@ -1145,6 +1139,721 @@ export function main(msg: string): never {
         assert!(results[0].is_err());
         assert_eq!(results[0], Err("test error".to_string()));
     }
+
+    // ==================== Multi-Script (Runner Group) Tests ====================
+
+    /// Job to send to a specific script in a multi-script wrapper
+    struct MultiScriptJob {
+        script_path: String,
+        args: serde_json::Value,
+    }
+
+    /// Creates a multi-script wrapper with multiple scripts as flat files, returns the wrapper path
+    fn create_multi_script_worker_files(
+        dir: &std::path::Path,
+        scripts: &[(&str, &str)], // (original_path, script_content)
+    ) -> std::path::PathBuf {
+        let mut entries_data = Vec::new();
+        for (path, content) in scripts {
+            let safe_name = format!("_wm_{}", path.replace('/', "__"));
+            std::fs::write(dir.join(format!("{safe_name}.ts")), content).unwrap();
+            entries_data.push((safe_name, path.to_string(), compute_ts_codegen(content)));
+        }
+
+        let entries: Vec<TsScriptEntry<'_>> = entries_data
+            .iter()
+            .map(|(safe, path, cg)| TsScriptEntry {
+                import_name: safe.as_str(),
+                original_path: path.as_str(),
+                codegen: cg,
+            })
+            .collect();
+
+        let wrapper = generate_multi_script_wrapper(&entries, "ts");
+        let wrapper_path = dir.join("wrapper.mjs");
+        std::fs::write(&wrapper_path, &wrapper).unwrap();
+        wrapper_path
+    }
+
+    /// Helper to run a multi-script dedicated worker test
+    fn run_multi_script_worker_test(
+        scripts: &[(&str, &str)],
+        jobs: Vec<MultiScriptJob>,
+    ) -> Vec<Result<serde_json::Value, String>> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let wrapper_path = create_multi_script_worker_files(temp_dir.path(), scripts);
+        let wrapper_str = wrapper_path.to_str().unwrap();
+
+        let mut cmd_args: Vec<&str> = BUN_DEDICATED_WORKER_ARGS.to_vec();
+        cmd_args.push(wrapper_str);
+
+        let mut child = Command::new(BUN_PATH.as_str())
+            .args(cmd_args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn worker process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        // Wait for "start" signal
+        let mut start_line = String::new();
+        reader.read_line(&mut start_line).unwrap();
+        assert_eq!(
+            parse_dedicated_worker_line(start_line.trim()),
+            DedicatedWorkerResult::Start,
+            "Expected 'start', got: {}",
+            start_line.trim()
+        );
+
+        let mut results = Vec::new();
+
+        for job in &jobs {
+            writeln!(stdin, "exec:{}:{}", job.script_path, job.args.to_string()).unwrap();
+            stdin.flush().unwrap();
+
+            let mut response = String::new();
+            reader.read_line(&mut response).unwrap();
+
+            match parse_dedicated_worker_line(response.trim()) {
+                DedicatedWorkerResult::Success(value) => results.push(Ok(value)),
+                DedicatedWorkerResult::Error(err) => {
+                    let msg = err["message"]
+                        .as_str()
+                        .unwrap_or("Unknown error")
+                        .to_string();
+                    results.push(Err(msg));
+                }
+                other => panic!("Unexpected response: {:?}", other),
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+
+        results
+    }
+
+    #[test]
+    fn test_multi_script_routing_basic() {
+        let script_add = r#"
+export function main(a: number, b: number): number {
+    return a + b;
+}
+"#;
+        let script_mul = r#"
+export function main(x: number, y: number): number {
+    return x * y;
+}
+"#;
+        let results = run_multi_script_worker_test(
+            &[("f/math/add", script_add), ("f/math/mul", script_mul)],
+            vec![
+                MultiScriptJob {
+                    script_path: "f/math/add".to_string(),
+                    args: serde_json::json!({"a": 3, "b": 4}),
+                },
+                MultiScriptJob {
+                    script_path: "f/math/mul".to_string(),
+                    args: serde_json::json!({"x": 5, "y": 6}),
+                },
+                // Route back to add
+                MultiScriptJob {
+                    script_path: "f/math/add".to_string(),
+                    args: serde_json::json!({"a": 10, "b": 20}),
+                },
+            ],
+        );
+
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0], Ok(serde_json::json!(7))); // 3 + 4
+        assert_eq!(results[1], Ok(serde_json::json!(30))); // 5 * 6
+        assert_eq!(results[2], Ok(serde_json::json!(30))); // 10 + 20
+    }
+
+    #[test]
+    fn test_multi_script_interleaved_jobs() {
+        let script_upper = r#"
+export function main(s: string): string {
+    return s.toUpperCase();
+}
+"#;
+        let script_len = r#"
+export function main(s: string): number {
+    return s.length;
+}
+"#;
+        let results = run_multi_script_worker_test(
+            &[("f/str/upper", script_upper), ("f/str/len", script_len)],
+            vec![
+                MultiScriptJob {
+                    script_path: "f/str/upper".to_string(),
+                    args: serde_json::json!({"s": "hello"}),
+                },
+                MultiScriptJob {
+                    script_path: "f/str/len".to_string(),
+                    args: serde_json::json!({"s": "hello"}),
+                },
+                MultiScriptJob {
+                    script_path: "f/str/upper".to_string(),
+                    args: serde_json::json!({"s": "world"}),
+                },
+                MultiScriptJob {
+                    script_path: "f/str/len".to_string(),
+                    args: serde_json::json!({"s": "ab"}),
+                },
+            ],
+        );
+
+        assert_eq!(results.len(), 4);
+        assert_eq!(results[0], Ok(serde_json::json!("HELLO")));
+        assert_eq!(results[1], Ok(serde_json::json!(5)));
+        assert_eq!(results[2], Ok(serde_json::json!("WORLD")));
+        assert_eq!(results[3], Ok(serde_json::json!(2)));
+    }
+
+    #[test]
+    fn test_multi_script_unknown_path_error() {
+        let script = r#"
+export function main(x: number): number {
+    return x;
+}
+"#;
+        let results = run_multi_script_worker_test(
+            &[("f/known", script)],
+            vec![MultiScriptJob {
+                script_path: "f/unknown".to_string(),
+                args: serde_json::json!({"x": 1}),
+            }],
+        );
+
+        assert_eq!(results.len(), 1);
+        assert!(results[0].is_err());
+        assert!(results[0]
+            .as_ref()
+            .unwrap_err()
+            .contains("Script not found"));
+    }
+
+    #[test]
+    fn test_multi_script_error_doesnt_break_other_scripts() {
+        let script_ok = r#"
+export function main(x: number): number {
+    return x * 2;
+}
+"#;
+        let script_err = r#"
+export function main(msg: string): never {
+    throw new Error(msg);
+}
+"#;
+        let results = run_multi_script_worker_test(
+            &[("f/ok", script_ok), ("f/err", script_err)],
+            vec![
+                MultiScriptJob {
+                    script_path: "f/ok".to_string(),
+                    args: serde_json::json!({"x": 5}),
+                },
+                MultiScriptJob {
+                    script_path: "f/err".to_string(),
+                    args: serde_json::json!({"msg": "boom"}),
+                },
+                // Should still work after error in other script
+                MultiScriptJob {
+                    script_path: "f/ok".to_string(),
+                    args: serde_json::json!({"x": 10}),
+                },
+            ],
+        );
+
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0], Ok(serde_json::json!(10)));
+        assert!(results[1].is_err());
+        assert_eq!(results[1], Err("boom".to_string()));
+        assert_eq!(results[2], Ok(serde_json::json!(20)));
+    }
+
+    // ==================== exec_preprocess Tests ====================
+
+    /// Raw protocol command to send to a dedicated worker
+    enum ProtocolCmd {
+        Exec { path: String, args: serde_json::Value },
+        ExecPreprocess { path: String, args: serde_json::Value },
+    }
+
+    /// Run a multi-script worker test with raw protocol commands, returning all protocol lines
+    fn run_raw_protocol_test(
+        scripts: &[(&str, &str)],
+        commands: Vec<ProtocolCmd>,
+    ) -> Vec<DedicatedWorkerResult> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let wrapper_path = create_multi_script_worker_files(temp_dir.path(), scripts);
+        let wrapper_str = wrapper_path.to_str().unwrap();
+
+        let mut cmd_args: Vec<&str> = BUN_DEDICATED_WORKER_ARGS.to_vec();
+        cmd_args.push(wrapper_str);
+
+        let mut child = Command::new(BUN_PATH.as_str())
+            .args(cmd_args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn worker process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        let mut start_line = String::new();
+        reader.read_line(&mut start_line).unwrap();
+        assert_eq!(
+            parse_dedicated_worker_line(start_line.trim()),
+            DedicatedWorkerResult::Start,
+        );
+
+        let mut results = Vec::new();
+
+        for cmd in &commands {
+            let line = match cmd {
+                ProtocolCmd::Exec { path, args } => format!("exec:{}:{}", path, args),
+                ProtocolCmd::ExecPreprocess { path, args } => {
+                    format!("exec_preprocess:{}:{}", path, args)
+                }
+            };
+            writeln!(stdin, "{}", line).unwrap();
+            stdin.flush().unwrap();
+
+            // exec_preprocess produces 2 response lines (preprocessed_args + success/error)
+            // exec produces 1 response line (success/error)
+            let expected_lines = match cmd {
+                ProtocolCmd::ExecPreprocess { .. } => 2,
+                ProtocolCmd::Exec { .. } => 1,
+            };
+
+            for _ in 0..expected_lines {
+                let mut response = String::new();
+                reader.read_line(&mut response).unwrap();
+                let parsed = parse_dedicated_worker_line(response.trim());
+                // If it's an error, stop reading more lines for this command
+                if matches!(parsed, DedicatedWorkerResult::Error(_)) {
+                    results.push(parsed);
+                    break;
+                }
+                results.push(parsed);
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+
+        results
+    }
+
+    #[test]
+    fn test_bun_exec_preprocess() {
+        let script = r#"
+export function preprocessor(x: number) {
+    return { x: x * 10 };
+}
+export function main(x: number): number {
+    return x + 1;
+}
+"#;
+        let results = run_raw_protocol_test(
+            &[("f/test/pre", script)],
+            vec![ProtocolCmd::ExecPreprocess {
+                path: "f/test/pre".to_string(),
+                args: serde_json::json!({"x": 5}),
+            }],
+        );
+        // Should get preprocessed_args then success
+        assert_eq!(results.len(), 2);
+        assert_eq!(
+            results[0],
+            DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50}))
+        );
+        // main(50) => 51
+        assert_eq!(
+            results[1],
+            DedicatedWorkerResult::Success(serde_json::json!(51))
+        );
+    }
+
+    #[test]
+    fn test_bun_exec_preprocess_missing_preprocessor() {
+        let script = r#"
+export function main(x: number): number {
+    return x;
+}
+"#;
+        let results = run_raw_protocol_test(
+            &[("f/test/nopre", script)],
+            vec![ProtocolCmd::ExecPreprocess {
+                path: "f/test/nopre".to_string(),
+                args: serde_json::json!({"x": 5}),
+            }],
+        );
+        assert_eq!(results.len(), 1);
+        assert!(matches!(results[0], DedicatedWorkerResult::Error(_)));
+    }
+
+    #[test]
+    fn test_bun_exec_preprocess_then_exec() {
+        let script = r#"
+export function preprocessor(x: number) {
+    return { x: x * 2 };
+}
+export function main(x: number): number {
+    return x + 100;
+}
+"#;
+        let results = run_raw_protocol_test(
+            &[("f/test/mixed", script)],
+            vec![
+                ProtocolCmd::ExecPreprocess {
+                    path: "f/test/mixed".to_string(),
+                    args: serde_json::json!({"x": 5}),
+                },
+                ProtocolCmd::Exec {
+                    path: "f/test/mixed".to_string(),
+                    args: serde_json::json!({"x": 7}),
+                },
+            ],
+        );
+        // preprocess: preprocessor(5) => {"x":10}, main(10) => 110
+        // exec: main(7) => 107
+        assert_eq!(results.len(), 3);
+        assert_eq!(
+            results[0],
+            DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10}))
+        );
+        assert_eq!(
+            results[1],
+            DedicatedWorkerResult::Success(serde_json::json!(110))
+        );
+        assert_eq!(
+            results[2],
+            DedicatedWorkerResult::Success(serde_json::json!(107))
+        );
+    }
+
+    // ==================== Argument Transformation Tests ====================
+
+    #[test]
+    fn test_bun_date_arg_transformation() {
+        let script = r#"
+export function main(d: Date): string {
+    return d instanceof Date ? d.toISOString() : typeof d;
+}
+"#;
+        let results = run_worker_test(
+            "bun",
+            script,
+            vec![serde_json::json!({"d": "2024-01-15T10:30:00.000Z"})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(
+            results[0],
+            Ok(serde_json::json!("2024-01-15T10:30:00.000Z"))
+        );
+    }
+
+    #[test]
+    fn test_bun_null_and_undefined_args() {
+        let script = r#"
+export function main(x?: number): string {
+    return x === null ? "null" : x === undefined ? "undefined" : String(x);
+}
+"#;
+        let results = run_worker_test(
+            "bun",
+            script,
+            vec![
+                serde_json::json!({"x": null}),
+                serde_json::json!({"x": 42}),
+                serde_json::json!({}),
+            ],
+        );
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0], Ok(serde_json::json!("null")));
+        assert_eq!(results[1], Ok(serde_json::json!("42")));
+        // Missing arg should be undefined
+        assert_eq!(results[2], Ok(serde_json::json!("undefined")));
+    }
+}
+
+// ============================================================================
+// Deno Dedicated Worker Protocol Tests
+// ============================================================================
+
+mod dedicated_worker_protocol_deno {
+    use std::io::{BufRead, BufReader, Write};
+    use std::process::{Command, Stdio};
+    use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
+    use windmill_worker::{generate_deno_dedicated_worker_wrapper, DENO_PATH};
+
+    const TEST_SCRIPT_PATH: &str = "f/test/script";
+
+    fn run_deno_worker_test(
+        script: &str,
+        jobs: Vec<serde_json::Value>,
+    ) -> Vec<Result<serde_json::Value, String>> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        std::fs::write(temp_dir.path().join("main.ts"), script).unwrap();
+
+        let wrapper = generate_deno_dedicated_worker_wrapper(script).unwrap();
+        std::fs::write(temp_dir.path().join("wrapper.ts"), &wrapper).unwrap();
+
+        let mut child = Command::new(DENO_PATH.as_str())
+            .args([
+                "run",
+                "--no-check",
+                "--unstable-unsafe-proto",
+                "--unstable-bare-node-builtins",
+                "-A",
+                "wrapper.ts",
+            ])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn deno process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        // Wait for "start" — deno outputs 'start\n' via console.log which adds
+        // its own newline, producing double newlines. Skip empty lines.
+        loop {
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            if line.trim().is_empty() {
+                continue;
+            }
+            assert_eq!(
+                parse_dedicated_worker_line(line.trim()),
+                DedicatedWorkerResult::Start,
+                "Expected 'start', got: {}",
+                line.trim()
+            );
+            break;
+        }
+
+        let mut results = Vec::new();
+        for job_args in jobs {
+            writeln!(stdin, "exec:{}:{}", TEST_SCRIPT_PATH, job_args.to_string()).unwrap();
+            stdin.flush().unwrap();
+
+            loop {
+                let mut response = String::new();
+                reader.read_line(&mut response).unwrap();
+                let trimmed = response.trim();
+                if trimmed.is_empty() {
+                    continue;
+                }
+                match parse_dedicated_worker_line(trimmed) {
+                    DedicatedWorkerResult::Success(value) => results.push(Ok(value)),
+                    DedicatedWorkerResult::Error(err) => {
+                        let msg = err["message"]
+                            .as_str()
+                            .unwrap_or("Unknown error")
+                            .to_string();
+                        results.push(Err(msg));
+                    }
+                    other => panic!("Unexpected response: {:?}", other),
+                }
+                break;
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+        results
+    }
+
+    #[test]
+    fn test_deno_dedicated_worker_simple() {
+        let script = r#"
+export function main(x: number, y: number): number {
+    return x + y;
+}
+"#;
+        let results = run_deno_worker_test(script, vec![serde_json::json!({"x": 5, "y": 3})]);
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0], Ok(serde_json::json!(8)));
+    }
+
+    #[test]
+    fn test_deno_dedicated_worker_multiple_jobs() {
+        let script = r#"
+export function main(n: number): number {
+    return n * 2;
+}
+"#;
+        let jobs: Vec<serde_json::Value> = (1..=5).map(|i| serde_json::json!({"n": i})).collect();
+        let results = run_deno_worker_test(script, jobs);
+        assert_eq!(results.len(), 5);
+        for (i, result) in results.iter().enumerate() {
+            assert_eq!(*result, Ok(serde_json::json!(((i + 1) * 2) as i64)));
+        }
+    }
+
+    #[test]
+    fn test_deno_dedicated_worker_error() {
+        let script = r#"
+export function main(msg: string): never {
+    throw new Error(msg);
+}
+"#;
+        let results = run_deno_worker_test(script, vec![serde_json::json!({"msg": "test error"})]);
+        assert_eq!(results.len(), 1);
+        assert!(results[0].is_err());
+        assert_eq!(results[0], Err("test error".to_string()));
+    }
+
+    // ==================== exec_preprocess Tests ====================
+
+    /// Run a raw deno protocol test, reading all output lines per command
+    fn run_deno_raw_protocol_test(
+        script: &str,
+        commands: Vec<(&str, serde_json::Value)>, // ("exec" or "exec_preprocess", args)
+    ) -> Vec<DedicatedWorkerResult> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        std::fs::write(temp_dir.path().join("main.ts"), script).unwrap();
+
+        let wrapper = generate_deno_dedicated_worker_wrapper(script).unwrap();
+        std::fs::write(temp_dir.path().join("wrapper.ts"), &wrapper).unwrap();
+
+        let mut child = Command::new(DENO_PATH.as_str())
+            .args([
+                "run",
+                "--no-check",
+                "--unstable-unsafe-proto",
+                "--unstable-bare-node-builtins",
+                "-A",
+                "wrapper.ts",
+            ])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn deno process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        // Wait for start, skip empty lines
+        loop {
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            if line.trim().is_empty() {
+                continue;
+            }
+            assert_eq!(
+                parse_dedicated_worker_line(line.trim()),
+                DedicatedWorkerResult::Start,
+            );
+            break;
+        }
+
+        let mut results = Vec::new();
+
+        for (cmd, args) in &commands {
+            writeln!(stdin, "{}:{}:{}", cmd, TEST_SCRIPT_PATH, args).unwrap();
+            stdin.flush().unwrap();
+
+            let expected_lines = if *cmd == "exec_preprocess" { 2 } else { 1 };
+
+            for _ in 0..expected_lines {
+                loop {
+                    let mut response = String::new();
+                    reader.read_line(&mut response).unwrap();
+                    if response.trim().is_empty() {
+                        continue;
+                    }
+                    let parsed = parse_dedicated_worker_line(response.trim());
+                    if matches!(parsed, DedicatedWorkerResult::Error(_)) {
+                        results.push(parsed);
+                        break;
+                    }
+                    results.push(parsed);
+                    break;
+                }
+                // If last result was an error, don't read more lines for this command
+                if matches!(results.last(), Some(DedicatedWorkerResult::Error(_))) {
+                    break;
+                }
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+
+        results
+    }
+
+    #[test]
+    fn test_deno_exec_preprocess() {
+        let script = r#"
+export function preprocessor(x: number) {
+    return { x: x * 10 };
+}
+export function main(x: number): number {
+    return x + 1;
+}
+"#;
+        let results = run_deno_raw_protocol_test(
+            script,
+            vec![("exec_preprocess", serde_json::json!({"x": 5}))],
+        );
+        assert_eq!(results.len(), 2);
+        assert_eq!(
+            results[0],
+            DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50}))
+        );
+        assert_eq!(
+            results[1],
+            DedicatedWorkerResult::Success(serde_json::json!(51))
+        );
+    }
+
+    // Note: no "missing preprocessor" test for Deno because the wrapper only generates
+    // the exec_preprocess handler when the script actually has a preprocessor function.
+    // Without one, exec_preprocess messages are unrecognized (by design — Rust never sends them).
+
+    // ==================== Argument Transformation Tests ====================
+
+    #[test]
+    fn test_deno_date_arg_transformation() {
+        let script = r#"
+export function main(d: Date): string {
+    return d instanceof Date ? d.toISOString() : typeof d;
+}
+"#;
+        let results = run_deno_worker_test(
+            script,
+            vec![serde_json::json!({"d": "2024-01-15T10:30:00.000Z"})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(
+            results[0],
+            Ok(serde_json::json!("2024-01-15T10:30:00.000Z"))
+        );
+    }
 }
 
 // ============================================================================

backend/tests/fixtures/typechecked_python.sql

@@ -0,0 +1,20 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import inspect
+import sys
+
+def greet(name: str) -> str:
+    # Verify that __file__ is set on this module (same check typeguard does)
+    mod = sys.modules[__name__]
+    source_file = inspect.getfile(mod)
+    return f"Hello, {name}! from {source_file}"
+
+def main():
+    return greet("World")
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/typechecked_helper', 12349, 'python3', '');

backend/tests/otel.rs

@@ -0,0 +1,504 @@
+//! E2E tests for OpenTelemetry integration.
+//!
+//! Verify that metrics are recorded with correct names/values/attributes and
+//! spans are created with correct trace IDs, attributes, and status codes.
+//!
+//! Run with: cargo test --features enterprise,private,otel --test otel -- --test-threads=1
+
+#![cfg(all(feature = "otel", feature = "enterprise"))]
+
+use std::sync::{atomic::Ordering, Arc};
+
+use opentelemetry::global;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::{
+    metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider},
+    trace::{InMemorySpanExporter, SdkTracerProvider, SimpleSpanProcessor},
+};
+use windmill_common::otel_ee::*;
+use windmill_common::{OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED};
+
+// ── Global test infrastructure ──────────────────────────────────────────
+
+struct OtelTestState {
+    metric_exporter: InMemoryMetricExporter,
+    span_exporter: InMemorySpanExporter,
+    meter_provider: SdkMeterProvider,
+}
+
+static STATE: tokio::sync::OnceCell<Arc<OtelTestState>> = tokio::sync::OnceCell::const_new();
+
+async fn ensure_setup() -> Arc<OtelTestState> {
+    STATE
+        .get_or_init(|| async {
+            // Metrics: InMemoryMetricExporter + PeriodicReader (needs async tokio context)
+            let metric_exporter = InMemoryMetricExporter::default();
+            let reader = PeriodicReader::builder(metric_exporter.clone()).build();
+            let meter_provider = SdkMeterProvider::builder().with_reader(reader).build();
+            global::set_meter_provider(meter_provider.clone());
+            OTEL_METRICS_ENABLED.store(true, Ordering::SeqCst);
+
+            // Tracing: InMemorySpanExporter + SimpleSpanProcessor
+            let span_exporter = InMemorySpanExporter::default();
+            let tracer_provider = SdkTracerProvider::builder()
+                .with_span_processor(SimpleSpanProcessor::new(span_exporter.clone()))
+                .build();
+            let tracer = tracer_provider.tracer("windmill");
+            *TRACER.write().unwrap() = Some(tracer);
+            OTEL_TRACING_ENABLED.store(true, Ordering::SeqCst);
+
+            Arc::new(OtelTestState { metric_exporter, span_exporter, meter_provider })
+        })
+        .await
+        .clone()
+}
+
+// ── Metric helper: flush + collect ──────────────────────────────────────
+
+fn flush_and_get_metrics(
+    state: &OtelTestState,
+) -> Vec<opentelemetry_sdk::metrics::data::ResourceMetrics> {
+    state.meter_provider.force_flush().expect("flush failed");
+    state
+        .metric_exporter
+        .get_finished_metrics()
+        .expect("get_finished_metrics failed")
+}
+
+fn find_metric<'a>(
+    all: &'a [opentelemetry_sdk::metrics::data::ResourceMetrics],
+    name: &str,
+) -> Option<&'a opentelemetry_sdk::metrics::data::Metric> {
+    all.iter()
+        .flat_map(|rm| rm.scope_metrics())
+        .flat_map(|sm| sm.metrics())
+        .find(|m| m.name() == name)
+}
+
+fn metric_names(all: &[opentelemetry_sdk::metrics::data::ResourceMetrics]) -> Vec<String> {
+    all.iter()
+        .flat_map(|rm| rm.scope_metrics())
+        .flat_map(|sm| sm.metrics())
+        .map(|m| m.name().to_string())
+        .collect()
+}
+
+// ── Counter value helpers ───────────────────────────────────────────────
+
+fn sum_u64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<u64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::U64(MetricData::Sum(sum)) => {
+            Some(sum.data_points().map(|dp| dp.value()).sum())
+        }
+        _ => None,
+    }
+}
+
+fn gauge_i64_values(
+    metric: &opentelemetry_sdk::metrics::data::Metric,
+) -> Vec<(Vec<opentelemetry::KeyValue>, i64)> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::I64(MetricData::Gauge(gauge)) => gauge
+            .data_points()
+            .map(|dp| (dp.attributes().cloned().collect(), dp.value()))
+            .collect(),
+        _ => panic!("expected I64 Gauge metric"),
+    }
+}
+
+fn gauge_f64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<f64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Gauge(gauge)) => {
+            gauge.data_points().next().map(|dp| dp.value())
+        }
+        _ => None,
+    }
+}
+
+fn histogram_f64_count(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<u64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Histogram(hist)) => {
+            Some(hist.data_points().map(|dp| dp.count()).sum())
+        }
+        _ => None,
+    }
+}
+
+fn histogram_f64_sum(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<f64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Histogram(hist)) => {
+            Some(hist.data_points().map(|dp| dp.sum()).sum())
+        }
+        _ => None,
+    }
+}
+
+// ═══════════════════════════════════════════════════════════════════════
+// METRICS E2E TEST
+//
+// All metric assertions live in one test function because the PeriodicReader's
+// background task is tied to the tokio runtime that created it. Separate
+// #[tokio::test] functions each get their own runtime, and the reader becomes
+// disconnected after the first test's runtime is dropped.
+// ═══════════════════════════════════════════════════════════════════════
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_all_metrics_e2e() {
+    let state = ensure_setup().await;
+
+    // ── Counters ────────────────────────────────────────────────────
+
+    otel_incr_queue_push_count();
+    otel_incr_queue_push_count();
+    otel_incr_queue_push_count();
+    otel_incr_queue_delete_count();
+    otel_incr_queue_pull_count();
+    otel_incr_zombie_restart_count(7);
+    otel_incr_zombie_delete_count(3);
+    otel_incr_worker_execution_count("bun");
+    otel_incr_worker_execution_count("bun");
+    otel_incr_worker_execution_failed("go");
+    otel_incr_worker_started();
+
+    // ── Gauges ──────────────────────────────────────────────────────
+
+    otel_set_queue_count("python3", 42);
+    otel_set_queue_running_count("deno", 5);
+    otel_set_worker_busy("worker-test-1", 1);
+    otel_set_db_pool(5, 10, 20);
+    otel_set_health_db_latency(2.5);
+    otel_set_worker_uptime("w-uptime", 3600.0);
+    otel_set_health_status_phase("healthy");
+    otel_set_health_db_unresponsive(true);
+
+    // ── Histograms ──────────────────────────────────────────────────
+
+    otel_record_worker_execution_duration("python3", 1.5);
+    otel_record_worker_execution_duration("python3", 2.5);
+    otel_record_worker_pull_duration("w1", true, 0.05);
+    otel_record_worker_pull_duration("w1", false, 0.01);
+
+    // ── Flush and collect ───────────────────────────────────────────
+
+    let metrics = flush_and_get_metrics(&state);
+    let names = metric_names(&metrics);
+
+    // ── Verify all 20 metric names are present ──────────────────────
+
+    let expected = [
+        "windmill.queue.push_count",
+        "windmill.queue.delete_count",
+        "windmill.queue.pull_count",
+        "windmill.queue.zombie_restart_count",
+        "windmill.queue.zombie_delete_count",
+        "windmill.queue.count",
+        "windmill.queue.running_count",
+        "windmill.worker.execution_count",
+        "windmill.worker.execution_duration",
+        "windmill.worker.busy",
+        "windmill.worker.pull_duration",
+        "windmill.worker.execution_failed",
+        "windmill.db.pool.active",
+        "windmill.db.pool.idle",
+        "windmill.db.pool.max",
+        "windmill.health.db_latency",
+        "windmill.worker.started",
+        "windmill.worker.uptime",
+        "windmill.health.status",
+        "windmill.health.db_unresponsive",
+    ];
+    for name in expected {
+        assert!(
+            names.iter().any(|n| n == name),
+            "metric '{}' not found in {:?}",
+            name,
+            names
+        );
+    }
+
+    // ── Counter values ──────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.queue.push_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 3, "push_count should be >= 3");
+
+    let m = find_metric(&metrics, "windmill.queue.delete_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.queue.pull_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.queue.zombie_restart_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 7);
+
+    let m = find_metric(&metrics, "windmill.queue.zombie_delete_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 3);
+
+    let m = find_metric(&metrics, "windmill.worker.execution_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 2);
+
+    let m = find_metric(&metrics, "windmill.worker.execution_failed").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.worker.started").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    // ── Gauge values ────────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.queue.count").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "python3")
+        })
+        .expect("queue.count data point with tag=python3 not found");
+    assert_eq!(dp.1, 42);
+
+    let m = find_metric(&metrics, "windmill.queue.running_count").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "deno")
+        })
+        .expect("running_count data point with tag=deno not found");
+    assert_eq!(dp.1, 5);
+
+    let m = find_metric(&metrics, "windmill.worker.busy").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "worker" && kv.value.as_str() == "worker-test-1")
+        })
+        .expect("worker.busy data point with worker=worker-test-1 not found");
+    assert_eq!(dp.1, 1);
+
+    let m = find_metric(&metrics, "windmill.db.pool.active").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 5);
+    let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 10);
+    let m = find_metric(&metrics, "windmill.db.pool.max").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 20);
+
+    let m = find_metric(&metrics, "windmill.health.db_latency").unwrap();
+    assert!((gauge_f64_value(m).unwrap() - 2.5).abs() < f64::EPSILON);
+
+    let m = find_metric(&metrics, "windmill.worker.uptime").unwrap();
+    assert!((gauge_f64_value(m).unwrap() - 3600.0).abs() < f64::EPSILON);
+
+    let m = find_metric(&metrics, "windmill.health.db_unresponsive").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 1);
+
+    // ── Health status phase (all 3 phases) ──────────────────────────
+
+    let m = find_metric(&metrics, "windmill.health.status").unwrap();
+    let values = gauge_i64_values(m);
+    let healthy = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "healthy")
+        })
+        .expect("phase=healthy");
+    let degraded = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "degraded")
+        })
+        .expect("phase=degraded");
+    let unhealthy = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "unhealthy")
+        })
+        .expect("phase=unhealthy");
+    assert_eq!(healthy.1, 1);
+    assert_eq!(degraded.1, 0);
+    assert_eq!(unhealthy.1, 0);
+
+    // ── Histogram values ────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.worker.execution_duration").unwrap();
+    assert!(histogram_f64_count(m).unwrap() >= 2);
+    assert!(histogram_f64_sum(m).unwrap() >= 4.0);
+
+    let m = find_metric(&metrics, "windmill.worker.pull_duration").unwrap();
+    assert!(histogram_f64_count(m).unwrap() >= 2);
+}
+
+// ═══════════════════════════════════════════════════════════════════════
+// SPAN E2E TESTS
+// ═══════════════════════════════════════════════════════════════════════
+
+fn make_test_job(id: uuid::Uuid, parent: Option<uuid::Uuid>) -> windmill_queue::MiniPulledJob {
+    use windmill_types::jobs::JobKind;
+    let mut job = windmill_queue::MiniPulledJob::new_inline(
+        "test-workspace".to_string(),
+        None,
+        "test-user".to_string(),
+        "u/test-user".to_string(),
+        "test@example.com".to_string(),
+        Some("f/test/script".to_string()),
+        JobKind::Script,
+        None,
+        "deno".to_string(),
+        None,
+    );
+    job.id = id;
+    job.parent_job = parent;
+    job.started_at = Some(chrono::Utc::now());
+    job
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_created_on_success() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    assert_eq!(span.status, opentelemetry::trace::Status::Ok,);
+
+    // Verify attributes
+    let attrs: Vec<_> = span.attributes.iter().map(|kv| kv.key.as_str()).collect();
+    assert!(attrs.contains(&"job_id"), "missing job_id attribute");
+    assert!(
+        attrs.contains(&"workspace_id"),
+        "missing workspace_id attribute"
+    );
+    assert!(
+        attrs.contains(&"script_path"),
+        "missing script_path attribute"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_error_on_failure() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, false);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    match &span.status {
+        opentelemetry::trace::Status::Error { description } => {
+            assert_eq!(description.as_ref(), "Job failed");
+        }
+        other => panic!("expected Error status, got {:?}", other),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_trace_id_matches_uuid() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let expected_trace_id =
+        opentelemetry::trace::TraceId::from_bytes(job_id.as_u128().to_be_bytes());
+    assert_eq!(span.span_context.trace_id(), expected_trace_id);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_id_matches_uuid() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let expected_span_id =
+        opentelemetry::trace::SpanId::from_bytes(job_id.as_u64_pair().1.to_be_bytes());
+    assert_eq!(span.span_context.span_id(), expected_span_id);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_child_job_produces_no_span() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let parent_id = uuid::Uuid::new_v4();
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, Some(parent_id));
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let found = spans.iter().any(|s| s.name == "full_job");
+    assert!(!found, "child job should not produce a span");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_attributes_values() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let get_attr = |key: &str| -> String {
+        span.attributes
+            .iter()
+            .find(|kv| kv.key.as_str() == key)
+            .map(|kv| kv.value.as_str().to_string())
+            .unwrap_or_default()
+    };
+
+    assert_eq!(get_attr("job_id"), job_id.to_string());
+    assert_eq!(get_attr("workspace_id"), "test-workspace");
+    assert_eq!(get_attr("script_path"), "f/test/script");
+}

backend/tests/python_jobs.rs

@@ -1,9 +1,470 @@
 use serde_json::json;
+#[cfg(feature = "python")]
 use sqlx::postgres::Postgres;
+#[cfg(feature = "python")]
 use sqlx::Pool;
+#[cfg(feature = "python")]
 use windmill_common::scripts::ScriptLang;
 use windmill_test_utils::*;
 
+// ============================================================================
+// Dedicated Worker Protocol Tests (Python)
+// ============================================================================
+
+#[cfg(feature = "python")]
+mod dedicated_worker_protocol_python {
+    use std::io::{BufRead, BufReader, Write};
+    use std::process::{Command, Stdio};
+    use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult};
+    use windmill_worker::{compute_py_codegen, generate_py_multi_script_wrapper, PyScriptEntry};
+
+    struct MultiScriptJob {
+        script_path: String,
+        args: serde_json::Value,
+    }
+
+    /// Creates a multi-script Python wrapper, writes scripts to proper module paths
+    fn create_py_worker_files(
+        dir: &std::path::Path,
+        scripts: &[(&str, &str)], // (original_path, content)
+    ) -> std::path::PathBuf {
+        let mut codegens = Vec::new();
+        for (path, content) in scripts {
+            let cg = compute_py_codegen(content, path);
+            let module_dir = dir.join(&cg.dirs);
+            std::fs::create_dir_all(&module_dir).unwrap();
+            std::fs::write(module_dir.join(format!("{}.py", cg.module_name)), content).unwrap();
+            codegens.push((path.to_string(), cg));
+        }
+
+        let entries: Vec<PyScriptEntry<'_>> = codegens
+            .iter()
+            .map(|(path, cg)| PyScriptEntry { original_path: path.as_str(), codegen: cg })
+            .collect();
+
+        let wrapper = generate_py_multi_script_wrapper(&entries, false, false);
+        let wrapper_path = dir.join("wrapper.py");
+        std::fs::write(&wrapper_path, &wrapper).unwrap();
+        wrapper_path
+    }
+
+    fn run_py_multi_script_test(
+        scripts: &[(&str, &str)],
+        jobs: Vec<MultiScriptJob>,
+    ) -> Vec<Result<serde_json::Value, String>> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        create_py_worker_files(temp_dir.path(), scripts);
+
+        let mut child = Command::new("python3")
+            .args(["-u", "-m", "wrapper"])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn python3 process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        let mut start_line = String::new();
+        reader.read_line(&mut start_line).unwrap();
+        assert_eq!(
+            parse_dedicated_worker_line(start_line.trim()),
+            DedicatedWorkerResult::Start,
+            "Expected 'start', got: {}",
+            start_line.trim()
+        );
+
+        let mut results = Vec::new();
+        for job in &jobs {
+            writeln!(stdin, "exec:{}:{}", job.script_path, job.args.to_string()).unwrap();
+            stdin.flush().unwrap();
+
+            let mut response = String::new();
+            reader.read_line(&mut response).unwrap();
+
+            match parse_dedicated_worker_line(response.trim()) {
+                DedicatedWorkerResult::Success(value) => results.push(Ok(value)),
+                DedicatedWorkerResult::Error(err) => {
+                    let msg = err["message"]
+                        .as_str()
+                        .unwrap_or("Unknown error")
+                        .to_string();
+                    results.push(Err(msg));
+                }
+                other => panic!("Unexpected response: {:?}", other),
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+        results
+    }
+
+    fn run_py_single_script_test(
+        script_path: &str,
+        content: &str,
+        jobs: Vec<serde_json::Value>,
+    ) -> Vec<Result<serde_json::Value, String>> {
+        run_py_multi_script_test(
+            &[(script_path, content)],
+            jobs.into_iter()
+                .map(|args| MultiScriptJob { script_path: script_path.to_string(), args })
+                .collect(),
+        )
+    }
+
+    #[test]
+    fn test_python_dedicated_worker_simple() {
+        let results = run_py_single_script_test(
+            "f/test/add",
+            "def main(a: int, b: int):\n    return a + b\n",
+            vec![serde_json::json!({"a": 3, "b": 4})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0], Ok(serde_json::json!(7)));
+    }
+
+    #[test]
+    fn test_python_dedicated_worker_multiple_jobs() {
+        let results = run_py_single_script_test(
+            "f/test/double",
+            "def main(n: int):\n    return n * 2\n",
+            (1..=5).map(|i| serde_json::json!({"n": i})).collect(),
+        );
+        assert_eq!(results.len(), 5);
+        for (i, result) in results.iter().enumerate() {
+            assert_eq!(*result, Ok(serde_json::json!(((i + 1) * 2) as i64)));
+        }
+    }
+
+    #[test]
+    fn test_python_multi_script_routing() {
+        let results = run_py_multi_script_test(
+            &[
+                (
+                    "f/math/add",
+                    "def main(a: int, b: int):\n    return a + b\n",
+                ),
+                (
+                    "f/math/mul",
+                    "def main(x: int, y: int):\n    return x * y\n",
+                ),
+            ],
+            vec![
+                MultiScriptJob {
+                    script_path: "f/math/add".to_string(),
+                    args: serde_json::json!({"a": 3, "b": 4}),
+                },
+                MultiScriptJob {
+                    script_path: "f/math/mul".to_string(),
+                    args: serde_json::json!({"x": 5, "y": 6}),
+                },
+                MultiScriptJob {
+                    script_path: "f/math/add".to_string(),
+                    args: serde_json::json!({"a": 10, "b": 20}),
+                },
+            ],
+        );
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0], Ok(serde_json::json!(7)));
+        assert_eq!(results[1], Ok(serde_json::json!(30)));
+        assert_eq!(results[2], Ok(serde_json::json!(30)));
+    }
+
+    #[test]
+    fn test_python_multi_script_error_isolation() {
+        let results = run_py_multi_script_test(
+            &[
+                ("f/ok", "def main(x: int):\n    return x * 2\n"),
+                ("f/err", "def main(msg: str):\n    raise Exception(msg)\n"),
+            ],
+            vec![
+                MultiScriptJob {
+                    script_path: "f/ok".to_string(),
+                    args: serde_json::json!({"x": 5}),
+                },
+                MultiScriptJob {
+                    script_path: "f/err".to_string(),
+                    args: serde_json::json!({"msg": "boom"}),
+                },
+                MultiScriptJob {
+                    script_path: "f/ok".to_string(),
+                    args: serde_json::json!({"x": 10}),
+                },
+            ],
+        );
+        assert_eq!(results.len(), 3);
+        assert_eq!(results[0], Ok(serde_json::json!(10)));
+        assert!(results[1].is_err());
+        assert_eq!(results[1], Err("boom".to_string()));
+        assert_eq!(results[2], Ok(serde_json::json!(20)));
+    }
+
+    #[test]
+    fn test_python_multi_script_unknown_path() {
+        let results = run_py_multi_script_test(
+            &[("f/known", "def main(x: int):\n    return x\n")],
+            vec![MultiScriptJob {
+                script_path: "f/unknown".to_string(),
+                args: serde_json::json!({"x": 1}),
+            }],
+        );
+        assert_eq!(results.len(), 1);
+        assert!(results[0].is_err());
+        assert!(results[0]
+            .as_ref()
+            .unwrap_err()
+            .contains("Script not found"));
+    }
+
+    // ==================== exec_preprocess Tests ====================
+
+    /// Raw protocol command for Python
+    enum ProtocolCmd {
+        Exec { path: String, args: serde_json::Value },
+        ExecPreprocess { path: String, args: serde_json::Value },
+    }
+
+    /// Run a Python worker test with raw protocol commands
+    fn run_py_raw_protocol_test(
+        scripts: &[(&str, &str)],
+        commands: Vec<ProtocolCmd>,
+    ) -> Vec<DedicatedWorkerResult> {
+        let temp_dir = tempfile::tempdir().unwrap();
+        create_py_worker_files(temp_dir.path(), scripts);
+
+        let mut child = Command::new("python3")
+            .args(["-u", "-m", "wrapper"])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .current_dir(temp_dir.path())
+            .spawn()
+            .expect("Failed to spawn python3 process");
+
+        let mut stdin = child.stdin.take().unwrap();
+        let stdout = child.stdout.take().unwrap();
+        let mut reader = BufReader::new(stdout);
+
+        let mut start_line = String::new();
+        reader.read_line(&mut start_line).unwrap();
+        assert_eq!(
+            parse_dedicated_worker_line(start_line.trim()),
+            DedicatedWorkerResult::Start,
+        );
+
+        let mut results = Vec::new();
+
+        for cmd in &commands {
+            let line = match cmd {
+                ProtocolCmd::Exec { path, args } => format!("exec:{}:{}", path, args),
+                ProtocolCmd::ExecPreprocess { path, args } => {
+                    format!("exec_preprocess:{}:{}", path, args)
+                }
+            };
+            writeln!(stdin, "{}", line).unwrap();
+            stdin.flush().unwrap();
+
+            let expected_lines = match cmd {
+                ProtocolCmd::ExecPreprocess { .. } => 2,
+                ProtocolCmd::Exec { .. } => 1,
+            };
+
+            for _ in 0..expected_lines {
+                let mut response = String::new();
+                reader.read_line(&mut response).unwrap();
+                let parsed = parse_dedicated_worker_line(response.trim());
+                if matches!(parsed, DedicatedWorkerResult::Error(_)) {
+                    results.push(parsed);
+                    break;
+                }
+                results.push(parsed);
+            }
+        }
+
+        writeln!(stdin, "end").unwrap();
+        stdin.flush().unwrap();
+        let _ = child.wait().expect("Worker process failed to exit");
+
+        results
+    }
+
+    #[test]
+    fn test_python_exec_preprocess() {
+        let script = r#"
+def preprocessor(x: int):
+    return {"x": x * 10}
+
+def main(x: int):
+    return x + 1
+"#;
+        let results = run_py_raw_protocol_test(
+            &[("f/test/pre", script)],
+            vec![ProtocolCmd::ExecPreprocess {
+                path: "f/test/pre".to_string(),
+                args: serde_json::json!({"x": 5}),
+            }],
+        );
+        assert_eq!(results.len(), 2);
+        assert_eq!(
+            results[0],
+            DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50}))
+        );
+        // main(50) => 51
+        assert_eq!(
+            results[1],
+            DedicatedWorkerResult::Success(serde_json::json!(51))
+        );
+    }
+
+    #[test]
+    fn test_python_exec_preprocess_missing_preprocessor() {
+        let script = "def main(x: int):\n    return x\n";
+        let results = run_py_raw_protocol_test(
+            &[("f/test/nopre", script)],
+            vec![ProtocolCmd::ExecPreprocess {
+                path: "f/test/nopre".to_string(),
+                args: serde_json::json!({"x": 5}),
+            }],
+        );
+        assert_eq!(results.len(), 1);
+        assert!(matches!(results[0], DedicatedWorkerResult::Error(_)));
+    }
+
+    #[test]
+    fn test_python_exec_preprocess_then_exec() {
+        let script = r#"
+def preprocessor(x: int):
+    return {"x": x * 2}
+
+def main(x: int):
+    return x + 100
+"#;
+        let results = run_py_raw_protocol_test(
+            &[("f/test/mixed", script)],
+            vec![
+                ProtocolCmd::ExecPreprocess {
+                    path: "f/test/mixed".to_string(),
+                    args: serde_json::json!({"x": 5}),
+                },
+                ProtocolCmd::Exec {
+                    path: "f/test/mixed".to_string(),
+                    args: serde_json::json!({"x": 7}),
+                },
+            ],
+        );
+        // preprocess: preprocessor(5) => {"x":10}, main(10) => 110
+        // exec: main(7) => 107
+        assert_eq!(results.len(), 3);
+        assert_eq!(
+            results[0],
+            DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10}))
+        );
+        assert_eq!(
+            results[1],
+            DedicatedWorkerResult::Success(serde_json::json!(110))
+        );
+        assert_eq!(
+            results[2],
+            DedicatedWorkerResult::Success(serde_json::json!(107))
+        );
+    }
+
+    // ==================== Argument Transformation Tests ====================
+
+    #[test]
+    fn test_python_datetime_arg_transformation() {
+        let script = r#"
+from datetime import datetime
+
+def main(d: datetime):
+    return d.isoformat()
+"#;
+        let results = run_py_single_script_test(
+            "f/test/dt",
+            script,
+            vec![serde_json::json!({"d": "2024-01-15T10:30:00+00:00"})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(
+            results[0],
+            Ok(serde_json::json!("2024-01-15T10:30:00+00:00"))
+        );
+    }
+
+    #[test]
+    fn test_python_bytes_arg_transformation() {
+        let script = r#"
+def main(data: bytes):
+    return len(data)
+"#;
+        // base64 of "hello" is "aGVsbG8="
+        let results = run_py_single_script_test(
+            "f/test/bytes",
+            script,
+            vec![serde_json::json!({"data": "aGVsbG8="})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0], Ok(serde_json::json!(5)));
+    }
+
+    #[test]
+    fn test_python_kwargs_filtering() {
+        // Test that extra kwargs are filtered out and only declared args are passed
+        let script = "def main(a: int, b: int):\n    return a + b\n";
+        let results = run_py_single_script_test(
+            "f/test/kwargs",
+            script,
+            vec![serde_json::json!({"a": 1, "b": 2, "extra": 99})],
+        );
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0], Ok(serde_json::json!(3)));
+    }
+
+    #[test]
+    fn test_python_function_call_sentinel_removal() {
+        // Test that '<function call>' sentinel values are removed from args
+        let script = "def main(a: int, b: int = 10):\n    return a + b\n";
+        let results = run_py_single_script_test(
+            "f/test/sentinel",
+            script,
+            vec![serde_json::json!({"a": 5, "b": "<function call>"})],
+        );
+        assert_eq!(results.len(), 1);
+        // b should be removed (sentinel), default 10 used
+        assert_eq!(results[0], Ok(serde_json::json!(15)));
+    }
+
+    // ==================== Relative Import Tests ====================
+
+    #[test]
+    fn test_python_dedicated_worker_with_relative_import_detection() {
+        // Test that the wrapper includes 'import loader' when scripts have relative imports
+        let script_with_relative = "from f.helper import util\ndef main(x: int):\n    return x\n";
+        let cg = compute_py_codegen(script_with_relative, "f/test/rel");
+        let entries = [PyScriptEntry { original_path: "f/test/rel", codegen: &cg }];
+        let wrapper = generate_py_multi_script_wrapper(&entries, false, true);
+        assert!(
+            wrapper.contains("import loader"),
+            "wrapper should contain 'import loader' when any_relative_imports=true"
+        );
+
+        // Without relative imports
+        let script_no_relative = "def main(x: int):\n    return x\n";
+        let cg2 = compute_py_codegen(script_no_relative, "f/test/norel");
+        let entries2 = [PyScriptEntry { original_path: "f/test/norel", codegen: &cg2 }];
+        let wrapper2 = generate_py_multi_script_wrapper(&entries2, false, false);
+        assert!(
+            !wrapper2.contains("import loader"),
+            "wrapper should NOT contain 'import loader' when any_relative_imports=false"
+        );
+    }
+}
+
 #[cfg(feature = "python")]
 #[sqlx::test(fixtures("base", "lockfile_python"))]
 async fn test_requirements_python(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -462,3 +923,43 @@ async def main(item: str, qty: int, email: str):
     .await;
     Ok(())
 }
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "typechecked_python"))]
+async fn test_typechecked_decorator_python(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let content = r#"
+from f.system.typechecked_helper import greet
+
+def main():
+    return greet("World")
+"#
+    .to_owned();
+
+    let job = JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: Some("f/system/test_typechecked".to_string()),
+        language: ScriptLang::Python3,
+        lock: None,
+        concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
+            .into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        modules: None,
+    });
+
+    let result = run_job_in_new_worker_until_complete(&db, false, job, port)
+        .await
+        .json_result()
+        .unwrap();
+
+    let result_str = result.as_str().unwrap();
+    assert!(result_str.starts_with("Hello, World! from "), "unexpected result: {result_str}");
+    Ok(())
+}

backend/windmill-api-auth/src/auth.rs

@@ -1,7 +1,6 @@
 #[cfg(feature = "enterprise")]
 use crate::ee_oss::ExternalJwks;
 use axum::{
-    async_trait,
     extract::{FromRequestParts, OriginalUri, Query},
     Extension, Json,
 };
@@ -226,7 +225,15 @@ impl AuthCache {
                     t_hash,
                     w_id.as_ref(),
                 )
-                .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
+                .map(|x| {
+                    (
+                        x.owner,
+                        x.email,
+                        x.super_admin,
+                        x.scopes,
+                        x.label,
+                    )
+                })
                 .fetch_optional(&self.db)
                 .await
                 .ok()
@@ -235,7 +242,13 @@ impl AuthCache {
                 if let Some(user) = user_o {
                     let authed_o = {
                         match user {
-                            (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
+                            (
+                                Some(owner),
+                                Some(email),
+                                super_admin,
+                                _,
+                                label,
+                            ) if w_id.is_some() => {
                                 let username_override = username_override_from_label(label);
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
@@ -451,7 +464,11 @@ pub(crate) async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S)
         None => Extension::<Cookies>::from_request_parts(parts, state)
             .await
             .ok()
-            .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
+            .and_then(|cookies| {
+                cookies
+                    .get(COOKIE_NAME)
+                    .map(|c| c.value_trimmed().to_owned())
+            }),
     };
 
     #[derive(Deserialize)]
@@ -504,7 +521,6 @@ impl BruteForceCounter {
     }
 }
 
-#[async_trait]
 impl<S> FromRequestParts<S> for Tokened
 where
     S: Send + Sync,
@@ -535,7 +551,6 @@ where
     }
 }
 
-#[async_trait]
 impl<S> FromRequestParts<S> for OptTokened
 where
     S: Send + Sync,

backend/windmill-api-auth/src/lib.rs

@@ -12,8 +12,7 @@ pub mod ee;
 pub mod ee_oss;
 pub mod scopes;
 
-use axum::async_trait;
-use axum::extract::FromRequestParts;
+use axum::extract::{FromRequestParts, OptionalFromRequestParts};
 use http::request::Parts;
 
 use windmill_audit::audit_oss::AuditAuthorable;
@@ -345,7 +344,6 @@ pub async fn maybe_refresh_folders(
 
 // ------------ FromRequestParts impls (direct call to auth module) ------------
 
-#[async_trait]
 impl<S> FromRequestParts<S> for ApiAuthed
 where
     S: Send + Sync,
@@ -361,7 +359,24 @@ where
     }
 }
 
-#[async_trait]
+impl<S> OptionalFromRequestParts<S> for ApiAuthed
+where
+    S: Send + Sync,
+{
+    type Rejection = std::convert::Infallible;
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &S,
+    ) -> std::result::Result<Option<Self>, Self::Rejection> {
+        Ok(
+            <Self as FromRequestParts<S>>::from_request_parts(parts, state)
+                .await
+                .ok(),
+        )
+    }
+}
+
 impl<S> FromRequestParts<S> for OptJobAuthed
 where
     S: Send + Sync,
@@ -397,7 +412,6 @@ fn empty_parts() -> Parts {
 #[derive(Clone, Debug)]
 pub struct OptAuthed(pub Option<ApiAuthed>);
 
-#[async_trait]
 impl<S> FromRequestParts<S> for OptAuthed
 where
     S: Send + Sync,
@@ -408,7 +422,7 @@ where
         parts: &mut Parts,
         state: &S,
     ) -> std::result::Result<Self, Self::Rejection> {
-        ApiAuthed::from_request_parts(parts, state)
+        <ApiAuthed as FromRequestParts<S>>::from_request_parts(parts, state)
             .await
             .map(|authed| Self(Some(authed)))
             .or_else(|_| Ok(Self(None)))

backend/windmill-api-configs/src/lib.rs

@@ -28,11 +28,11 @@ use windmill_api_auth::{require_devops_role, ApiAuthed};
 pub fn global_service() -> Router {
     Router::new()
         .route("/list_worker_groups", get(list_worker_groups))
-        .route("/update/:name", post(update_config).delete(delete_config))
-        .route("/get/:name", get(get_config))
+        .route("/update/{name}", post(update_config).delete(delete_config))
+        .route("/get/{name}", get(get_config))
         .route("/list", get(list_configs))
         .route(
-            "/list_autoscaling_events/:worker_group",
+            "/list_autoscaling_events/{worker_group}",
             get(list_autoscaling_events),
         )
         .route(

backend/windmill-api-debug/src/lib.rs

@@ -87,6 +87,7 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/sign", post(sign_debug_request))
         .route("/sign_expression", post(sign_expression))
+        .route("/sign_multiplayer", post(sign_multiplayer))
 }
 
 /// JWKS response containing the public key for debug token verification
@@ -416,3 +417,62 @@ async fn sign_expression(
 
     Ok(Json(SignedExpressionPayload { token }))
 }
+
+/// JWT claims for multiplayer session tokens
+#[derive(Serialize, Deserialize)]
+pub struct MultiplayerTokenClaims {
+    /// Workspace ID
+    pub workspace_id: String,
+    /// User email
+    pub email: String,
+    /// Issued at (Unix timestamp)
+    pub iat: i64,
+    /// Expiration (Unix timestamp)
+    pub exp: i64,
+    /// Token purpose (always "multiplayer")
+    pub purpose: String,
+}
+
+#[derive(Serialize)]
+pub struct SignedMultiplayerPayload {
+    pub token: String,
+}
+
+/// Sign a multiplayer session request.
+///
+/// Returns a JWT that the multiplayer server will verify using the public key from /api/debug/jwks.
+async fn sign_multiplayer(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+) -> JsonResult<SignedMultiplayerPayload> {
+    let key_guard = DEBUG_SIGNING_KEY.read().await;
+    let signing_key = key_guard.as_ref().ok_or_else(|| {
+        windmill_common::error::Error::InternalErr("Debug signing key not initialized".to_string())
+    })?;
+
+    let now_ts = Utc::now().timestamp();
+    let exp = now_ts + DEBUG_TOKEN_TTL_SECS;
+
+    let claims = MultiplayerTokenClaims {
+        workspace_id: w_id,
+        email: authed.email,
+        iat: now_ts,
+        exp,
+        purpose: "multiplayer".to_string(),
+    };
+
+    let header = serde_json::json!({
+        "alg": "EdDSA",
+        "typ": "JWT"
+    });
+    let header_b64 = URL_SAFE_NO_PAD.encode(serde_json::to_string(&header).unwrap());
+    let claims_b64 = URL_SAFE_NO_PAD.encode(serde_json::to_string(&claims).unwrap());
+    let message = format!("{}.{}", header_b64, claims_b64);
+
+    let signature = signing_key.sign(message.as_bytes());
+    let signature_b64 = URL_SAFE_NO_PAD.encode(signature.to_bytes());
+
+    let token = format!("{}.{}", message, signature_b64);
+
+    Ok(Json(SignedMultiplayerPayload { token }))
+}

backend/windmill-api-flow-conversations/src/lib.rs

@@ -21,8 +21,8 @@ use windmill_common::{
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_conversations))
-        .route("/delete/:conversation_id", delete(delete_conversation))
-        .route("/:conversation_id/messages", get(list_messages))
+        .route("/delete/{conversation_id}", delete(delete_conversation))
+        .route("/{conversation_id}/messages", get(list_messages))
 }
 
 #[derive(Serialize, FromRow, Debug)]

backend/windmill-api-flows/src/flows.rs

@@ -61,26 +61,26 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_flows))
         .route("/list_search", get(list_search_flows))
         .route("/create", post(create_flow))
-        .route("/update/*path", post(update_flow))
-        .route("/archive/*path", post(archive_flow_by_path))
-        .route("/delete/*path", delete(delete_flow_by_path))
-        .route("/list_tokens/*path", get(list_tokens))
-        .route("/get/*path", get(get_flow_by_path))
-        .route("/deployment_status/p/*path", get(get_deployment_status))
-        .route("/get/draft/*path", get(get_flow_by_path_w_draft))
-        .route("/exists/*path", get(exists_flow_by_path))
+        .route("/update/{*path}", post(update_flow))
+        .route("/archive/{*path}", post(archive_flow_by_path))
+        .route("/delete/{*path}", delete(delete_flow_by_path))
+        .route("/list_tokens/{*path}", get(list_tokens))
+        .route("/get/{*path}", get(get_flow_by_path))
+        .route("/deployment_status/p/{*path}", get(get_deployment_status))
+        .route("/get/draft/{*path}", get(get_flow_by_path_w_draft))
+        .route("/exists/{*path}", get(exists_flow_by_path))
         .route("/list_paths", get(list_paths))
-        .route("/history/p/*path", get(get_flow_history))
-        .route("/get_latest_version/*path", get(get_latest_version))
+        .route("/history/p/{*path}", get(get_flow_history))
+        .route("/get_latest_version/{*path}", get(get_latest_version))
         .route(
-            "/list_paths_from_workspace_runnable/:runnable_kind/*path",
+            "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}",
             get(list_paths_from_workspace_runnable),
         )
-        .route("/history_update/v/:version", post(update_flow_history))
-        .route("/get/v/:version", get(get_flow_version_by_id))
-        .route("/get/v/:version/p/*path", get(get_flow_version))
+        .route("/history_update/v/{version}", post(update_flow_history))
+        .route("/get/v/{version}", get(get_flow_version_by_id))
+        .route("/get/v/{version}/p/{*path}", get(get_flow_version))
         .route(
-            "/toggle_workspace_error_handler/*path",
+            "/toggle_workspace_error_handler/{*path}",
             post(toggle_workspace_error_handler),
         )
 }
@@ -88,7 +88,7 @@ pub fn workspaced_service() -> Router {
 pub fn global_service() -> Router {
     Router::new()
         .route("/hub/list", get(list_hub_flows))
-        .route("/hub/get/:id", get(get_hub_flow_by_id))
+        .route("/hub/get/{id}", get(get_hub_flow_by_id))
 }
 
 #[derive(Serialize, FromRow)]
@@ -1657,6 +1657,38 @@ async fn delete_flow_by_path(
     }
     let mut tx = user_db.begin(&authed).await?;
 
+    // Capture all related data for trashbin before deleting (CASCADE will remove flow_version, flow_node)
+    let trash_flow: Option<serde_json::Value> =
+        sqlx::query_scalar("SELECT to_jsonb(t) FROM flow t WHERE path = $1 AND workspace_id = $2")
+            .bind(path)
+            .bind(&w_id)
+            .fetch_optional(&mut *tx)
+            .await?;
+
+    let trash_flow_versions: Vec<serde_json::Value> = sqlx::query_scalar(
+        "SELECT to_jsonb(t) FROM flow_version t WHERE path = $1 AND workspace_id = $2",
+    )
+    .bind(path)
+    .bind(&w_id)
+    .fetch_all(&mut *tx)
+    .await?;
+
+    let trash_flow_nodes: Vec<serde_json::Value> = sqlx::query_scalar(
+        "SELECT to_jsonb(t) FROM flow_node t WHERE path = $1 AND workspace_id = $2",
+    )
+    .bind(path)
+    .bind(&w_id)
+    .fetch_all(&mut *tx)
+    .await?;
+
+    let trash_drafts: Vec<serde_json::Value> = sqlx::query_scalar(
+        "SELECT to_jsonb(t) FROM draft t WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+    )
+    .bind(path)
+    .bind(&w_id)
+    .fetch_all(&mut *tx)
+    .await?;
+
     sqlx::query!(
         "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
         path,
@@ -1673,6 +1705,28 @@ async fn delete_flow_by_path(
     .execute(&mut *tx)
     .await?;
 
+    if let Some(flow_data) = trash_flow {
+        let mut trash_data = serde_json::json!({"row": flow_data});
+        if !trash_flow_versions.is_empty() {
+            trash_data["flow_versions"] = serde_json::Value::Array(trash_flow_versions);
+        }
+        if !trash_flow_nodes.is_empty() {
+            trash_data["flow_nodes"] = serde_json::Value::Array(trash_flow_nodes);
+        }
+        if !trash_drafts.is_empty() {
+            trash_data["drafts"] = serde_json::Value::Array(trash_drafts);
+        }
+        windmill_common::trashbin::move_to_trash(
+            &mut *tx,
+            &w_id,
+            "flow",
+            path,
+            trash_data,
+            &authed.username,
+        )
+        .await?;
+    }
+
     if !query.keep_captures.unwrap_or(false) {
         sqlx::query!(
             "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE",

backend/windmill-api-groups/src/folder_history.rs

@@ -22,7 +22,7 @@ use serde::Serialize;
 use sqlx::FromRow;
 
 pub fn workspaced_service() -> Router {
-    Router::new().route("/get/:name", get(get_folder_permission_history))
+    Router::new().route("/get/{name}", get(get_folder_permission_history))
 }
 
 #[derive(Serialize, FromRow)]

backend/windmill-api-groups/src/folders.rs

@@ -40,14 +40,14 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_folders))
         .route("/listnames", get(list_foldernames))
         .route("/create", post(create_folder))
-        .route("/get/:name", get(get_folder))
-        .route("/exists/:name", get(exists_folder))
-        .route("/update/:name", post(update_folder))
-        .route("/getusage/:name", get(get_folder_usage))
-        .route("/delete/:name", delete(delete_folder))
-        .route("/addowner/:name", post(add_owner))
-        .route("/removeowner/:name", post(remove_owner))
-        .route("/is_owner/*path", get(is_owner_api))
+        .route("/get/{name}", get(get_folder))
+        .route("/exists/{name}", get(exists_folder))
+        .route("/update/{name}", post(update_folder))
+        .route("/getusage/{name}", get(get_folder_usage))
+        .route("/delete/{name}", delete(delete_folder))
+        .route("/addowner/{name}", post(add_owner))
+        .route("/removeowner/{name}", post(remove_owner))
+        .route("/is_owner/{*path}", get(is_owner_api))
 }
 
 #[derive(FromRow, Serialize, Deserialize, Clone)]

backend/windmill-api-groups/src/granular_acls.rs

@@ -48,9 +48,9 @@ const KINDS: [&str; 19] = [
 
 pub fn workspaced_service() -> Router {
     Router::new()
-        .route("/get/*path", get(get_granular_acls))
-        .route("/add/*path", post(add_granular_acl))
-        .route("/remove/*path", post(remove_granular_acl))
+        .route("/get/{*path}", get(get_granular_acls))
+        .route("/add/{*path}", post(add_granular_acl))
+        .route("/remove/{*path}", post(remove_granular_acl))
 }
 
 #[derive(Serialize, Deserialize)]

backend/windmill-api-groups/src/groups.rs

@@ -33,24 +33,24 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_groups))
         .route("/listnames", get(list_group_names))
         .route("/create", post(create_group))
-        .route("/get/:name", get(get_group))
-        .route("/update/:name", post(update_group))
-        .route("/delete/:name", delete(delete_group))
-        .route("/adduser/:name", post(add_user))
-        .route("/removeuser/:name", post(remove_user))
-        .route("/is_owner/:name", get(is_owner))
+        .route("/get/{name}", get(get_group))
+        .route("/update/{name}", post(update_group))
+        .route("/delete/{name}", delete(delete_group))
+        .route("/adduser/{name}", post(add_user))
+        .route("/removeuser/{name}", post(remove_user))
+        .route("/is_owner/{name}", get(is_owner))
 }
 
 pub fn global_service() -> Router {
     Router::new()
         .route("/list", get(list_igroups))
         .route("/list_with_workspaces", get(list_igroups_with_workspaces))
-        .route("/get/:name", get(get_igroup))
+        .route("/get/{name}", get(get_igroup))
         .route("/create", post(create_igroup))
-        .route("/update/:name", post(update_igroup))
-        .route("/delete/:name", delete(delete_igroup))
-        .route("/adduser/:name", post(add_user_igroup))
-        .route("/removeuser/:name", post(remove_user_igroup))
+        .route("/update/{name}", post(update_igroup))
+        .route("/delete/{name}", delete(delete_igroup))
+        .route("/adduser/{name}", post(add_user_igroup))
+        .route("/removeuser/{name}", post(remove_user_igroup))
         .route("/export", get(export_igroups))
         .route("/overwrite", post(overwrite_igroups))
 }
@@ -656,6 +656,7 @@ async fn delete_group(
     )
     .execute(&mut *tx)
     .await?;
+
     audit_log(
         &mut *tx,
         &authed,

backend/windmill-api-inputs/src/lib.rs

@@ -33,9 +33,9 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_saved_inputs))
         .route("/create", post(create_input))
         .route("/update", post(update_input))
-        .route("/delete/:id", post(delete_input))
+        .route("/delete/{id}", post(delete_input))
         .route(
-            "/:job_or_input_id/args",
+            "/{job_or_input_id}/args",
             get(get_args_from_history_or_saved_input),
         )
 }

backend/windmill-api-integration-tests/Cargo.toml

@@ -31,6 +31,7 @@ reqwest.workspace = true
 tokio.workspace = true
 anyhow.workspace = true
 uuid.workspace = true
+futures.workspace = true
 rand.workspace = true
 rumqttc.workspace = true
 rdkafka.workspace = true
@@ -39,3 +40,4 @@ aws-config = { workspace = true, optional = true }
 aws-credential-types = { workspace = true, optional = true }
 aws-sdk-sqs = { workspace = true, optional = true }
 base64 = { workspace = true, optional = true }
+axum.workspace = true

backend/windmill-api-integration-tests/tests/ai_routes.rs

@@ -0,0 +1,106 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+/// Start a mock AI API that echoes back a valid chat completion response.
+async fn start_mock_ai_api() -> u16 {
+    use axum::{routing::post, Json, Router};
+
+    let app = Router::new().fallback(post(|| async {
+        Json(json!({
+            "id": "chatcmpl-test",
+            "object": "chat.completion",
+            "choices": [{"message": {"role": "assistant", "content": "hello"}}]
+        }))
+    }));
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let port = listener.local_addr().unwrap().port();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    port
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_ai_proxy_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // Start mock AI API
+    let mock_port = start_mock_ai_api().await;
+    let mock_url = format!("http://127.0.0.1:{mock_port}/v1");
+
+    // Create an openai resource pointing to the mock
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/resources/create"
+            ))
+            .json(&json!({
+                "path": "f/ai/openai_config",
+                "resource_type": "openai",
+                "value": {
+                    "api_key": "test-key",
+                    "base_url": mock_url
+                }
+            })),
+    )
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "create openai resource",
+    );
+
+    // Set ai_config on workspace_settings directly via SQL
+    sqlx::query(
+        "UPDATE workspace_settings SET ai_config = $1::jsonb WHERE workspace_id = 'test-workspace'",
+    )
+    .bind(json!({
+        "providers": {
+            "openai": {
+                "resource_path": "f/ai/openai_config",
+                "models": ["gpt-4"]
+            }
+        }
+    }))
+    .execute(&db)
+    .await?;
+
+    // POST /w/{ws}/ai/proxy/chat/completions
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/ai/proxy/chat/completions"
+            ))
+            .header("X-Provider", "openai")
+            .json(&json!({
+                "model": "gpt-4",
+                "messages": [{"role": "user", "content": "hi"}]
+            })),
+    )
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /ai/proxy/chat/completions",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/audit.rs

@@ -0,0 +1,35 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_audit_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/audit");
+
+    // GET /list returns 200 (empty array)
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /audit/list",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/capture_unauthed.rs

@@ -0,0 +1,83 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_capture_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // POST /capture/set_config → 200 (authed)
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/capture/set_config"
+            ))
+            .json(&json!({
+                "trigger_kind": "webhook",
+                "path": "u/test-user/test_capture",
+                "is_flow": false
+            })),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /capture/set_config");
+
+    // GET /capture/list/{...} → 200 (authed)
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/list/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(
+        status,
+        &body,
+        "GET /capture/list/script/u/test-user/test_capture",
+    );
+
+    // POST /capture/ping_config/{trigger_kind}/{runnable_kind}/{*path} → 200
+    let resp = authed(client().post(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/ping_config/webhook/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /capture/ping_config",
+    );
+
+    // GET /capture/get_configs/{runnable_kind}/{*path} → 200
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/get_configs/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /capture/get_configs",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/concurrency_groups.rs

@@ -0,0 +1,48 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_concurrency_groups_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/concurrency_groups/list"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /api/concurrency_groups/list",
+    );
+
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/concurrency_groups/list_jobs"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /api/w/test-workspace/concurrency_groups/list_jobs",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/favorites.rs

@@ -0,0 +1,72 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_favorites_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+    // Setup: create a script to favorite
+    let resp = authed(client().post(format!("{ws}/scripts/create")))
+        .json(&json!({
+            "path": "u/test-user/test_fav_script",
+            "summary": "test",
+            "description": "",
+            "content": "export function main() { return 1; }",
+            "language": "deno",
+            "schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /scripts/create (setup)");
+
+    let fav_body = json!({
+        "favorite_kind": "script",
+        "path": "u/test-user/test_fav_script"
+    });
+
+    // POST /favorites/star → 200
+    let resp = authed(client().post(format!("{ws}/favorites/star")))
+        .json(&fav_body)
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /favorites/star");
+
+    // POST /favorites/unstar → 200
+    let resp = authed(client().post(format!("{ws}/favorites/unstar")))
+        .json(&fav_body)
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /favorites/unstar");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/folder_history.rs

@@ -0,0 +1,51 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_folder_history_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // Create a folder first
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/folders/create"
+            ))
+            .json(&json!({"name": "test_hist_folder", "owners": ["u/test-user"]})),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /folders/create");
+
+    // GET /folders_history/get/{folder} → 200 (empty array)
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/folders_history/get/test_hist_folder"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /folders_history/get/test_hist_folder");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/granular_acls.rs

@@ -0,0 +1,54 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_granular_acls_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/acls");
+
+    // GET /acls/get/group_/all → 200
+    let resp = authed(client().get(format!("{base}/get/group_/all")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /acls/get/group_/all");
+
+    // POST /acls/add/group_/all → 200
+    let resp = authed(client().post(format!("{base}/add/group_/all")))
+        .json(&json!({"owner": "u/test-user-2", "write": true}))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /acls/add/group_/all");
+
+    // POST /acls/remove/group_/all → 200
+    let resp = authed(client().post(format!("{base}/remove/group_/all")))
+        .json(&json!({"owner": "u/test-user-2"}))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /acls/remove/group_/all");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/group_history.rs

@@ -0,0 +1,32 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_group_history_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/groups_history");
+
+    let resp = authed(client().get(format!("{base}/get/all")))
+        .send()
+        .await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get/all");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/health.rs

@@ -0,0 +1,41 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_health_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/health");
+
+    // GET /health/status → 200 (no auth required)
+    let resp = client().get(format!("{base}/status")).send().await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /health/status");
+
+    // GET /health/detailed → 200 (authed)
+    let resp = authed(client().get(format!("{base}/detailed")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /health/detailed");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/inputs.rs

@@ -0,0 +1,76 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_inputs_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/inputs");
+
+    // GET /history with fake runnable → 200 empty array
+    let resp = authed(client().get(format!(
+        "{base}/history?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/history");
+
+    // GET /list with fake runnable → 200 empty array
+    let resp = authed(client().get(format!(
+        "{base}/list?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/list");
+
+    // POST /create → 200, returns UUID
+    let resp = authed(client().post(format!(
+        "{base}/create?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .json(&json!({"name": "test_input", "args": {}}))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /inputs/create");
+    let input_id: String = serde_json::from_str(&body)?;
+
+    // GET /{id}/args → 200
+    let resp = authed(client().get(format!("{base}/{input_id}/args")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/{id}/args");
+
+    // POST /delete/{id} → 200
+    let resp = authed(client().post(format!("{base}/delete/{input_id}")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /inputs/delete/{id}");
+
+    Ok(())
+}