Merge branch 'main' into py-typechecked-decorator

* chore(main): release 1.668.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>

.github/workflows/benchmark.yml

@@ -290,6 +290,49 @@ jobs:
           path: |
             *.json
 
+  benchmark_wac:
+    runs-on: ubicloud-standard-8
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+          POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+          --shm-size=2g
+      windmill:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+    steps:
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      - name: benchmark
+        timeout-minutes: 30
+        run: deno run  -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_wac.json
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_wac
+          path: |
+            *.json
+
   benchmark_graphs:
     runs-on: ubicloud
     needs:
@@ -297,6 +340,7 @@ jobs:
       - benchmark_dedicated
       - benchmark_4workers
       - benchmark_8workers
+      - benchmark_wac
     steps:
       - uses: denoland/setup-deno@v2
         with:

.github/workflows/cli-tests.yml

@@ -1,6 +1,7 @@
 name: CLI Tests
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:

CHANGELOG.md

@@ -1,5 +1,64 @@
 # Changelog
 
+## [1.668.2](https://github.com/windmill-labs/windmill/compare/v1.668.1...v1.668.2) (2026-03-28)
+
+
+### Bug Fixes
+
+* **cli:** app push crash, lint path, push --message, run validation, history timestamps ([#8585](https://github.com/windmill-labs/windmill/issues/8585)) ([f40cdaf](https://github.com/windmill-labs/windmill/commit/f40cdaf43453d2643800ed730d6abe6873bbe8e7))
+
+## [1.668.1](https://github.com/windmill-labs/windmill/compare/v1.668.0...v1.668.1) (2026-03-28)
+
+
+### Bug Fixes
+
+* **cli:** fix 13 CLI bugs — exit codes, sync tar fallback, variable encryption, JSON output ([#8582](https://github.com/windmill-labs/windmill/issues/8582)) ([38acaa3](https://github.com/windmill-labs/windmill/commit/38acaa3653728bf9e0ae6f746edf433703b4ab63))
+
+## [1.668.0](https://github.com/windmill-labs/windmill/compare/v1.667.0...v1.668.0) (2026-03-28)
+
+
+### Features
+
+* add DB health diagnostic dashboard for superadmins ([#8574](https://github.com/windmill-labs/windmill/issues/8574)) ([9ceab73](https://github.com/windmill-labs/windmill/commit/9ceab730d7def09c2b46527f8a586789d14f2ce0))
+* **cli:** add job, group, audit, token commands and schedule enable/disable ([#8581](https://github.com/windmill-labs/windmill/issues/8581)) ([d29cb23](https://github.com/windmill-labs/windmill/commit/d29cb234dbff07473b911e5e75e362def8a47650))
+* IAM RDS auth for PostgreSQL worker resources ([#8573](https://github.com/windmill-labs/windmill/issues/8573)) ([56253c0](https://github.com/windmill-labs/windmill/commit/56253c04cb679c58d00750da699a6cb62ed52aca))
+
+
+### Bug Fixes
+
+* add Authority Key Identifier to MITM proxy leaf certs ([#8576](https://github.com/windmill-labs/windmill/issues/8576)) ([ce2e6c8](https://github.com/windmill-labs/windmill/commit/ce2e6c8c015110d0385e6afecdc8313aabca1364))
+* Improve CLI developer experience: error handling, sync workflow, JSON output, workspace forks ([#8578](https://github.com/windmill-labs/windmill/issues/8578)) ([501a4ff](https://github.com/windmill-labs/windmill/commit/501a4ff2a94510145952686d24ccc639781beefe))
+* trigger capture filter and focus issues ([#8579](https://github.com/windmill-labs/windmill/issues/8579)) ([820f28f](https://github.com/windmill-labs/windmill/commit/820f28f8799f8dad5cfab94b51ac9921d664f04a))
+
+## [1.667.0](https://github.com/windmill-labs/windmill/compare/v1.666.0...v1.667.0) (2026-03-27)
+
+
+### Features
+
+* add schedule support to CLI branch-specific items ([#8570](https://github.com/windmill-labs/windmill/issues/8570)) ([b592996](https://github.com/windmill-labs/windmill/commit/b592996eee98ddb664f1b007b95a2096d5d4e3a6))
+* add workspace-level service accounts ([#8560](https://github.com/windmill-labs/windmill/issues/8560)) ([3959fe8](https://github.com/windmill-labs/windmill/commit/3959fe82974f5f0383e94fd83a5d78fe4212d56a))
+* **cli:** generate commented wmill.yaml and add config reference command ([#8546](https://github.com/windmill-labs/windmill/issues/8546)) ([d06b426](https://github.com/windmill-labs/windmill/commit/d06b42613f73c4a7b31c990be22b0c97efab2666))
+* DB-coordinated graceful restart staggering for settings changes ([#8555](https://github.com/windmill-labs/windmill/issues/8555)) ([2f32675](https://github.com/windmill-labs/windmill/commit/2f326758013dd1f1e6ae732e5784a32f1fb6e4bd))
+* improve-replay-ui ([#8250](https://github.com/windmill-labs/windmill/issues/8250)) ([c0aafee](https://github.com/windmill-labs/windmill/commit/c0aafee9a9923d5dc2fa3b99da4378e923933a06))
+* support multiple folder selection in MCP scope selector ([#8557](https://github.com/windmill-labs/windmill/issues/8557)) ([ad19ac9](https://github.com/windmill-labs/windmill/commit/ad19ac9b37b04591c921f93f180bdda961af6cef))
+
+
+### Bug Fixes
+
+* **cli:** preserve inline script files during flow generate-locks ([#8561](https://github.com/windmill-labs/windmill/issues/8561)) ([a8b651d](https://github.com/windmill-labs/windmill/commit/a8b651da9ff86766119e14c0b61652be8a7b453a))
+* emit 0 for OTEL queue metrics when tag queue is empty ([#8559](https://github.com/windmill-labs/windmill/issues/8559)) ([79cc4a9](https://github.com/windmill-labs/windmill/commit/79cc4a92d88486c999799826bd0c9663767103f5))
+* handle inline script deletion in sync push + flow new nonDottedPaths ([#8553](https://github.com/windmill-labs/windmill/issues/8553)) ([943fe9c](https://github.com/windmill-labs/windmill/commit/943fe9c6cc9b046e24007e45b5c37afc4804256a))
+* include importer_kind in dependency debounce key to prevent cross-kind collisions ([#8567](https://github.com/windmill-labs/windmill/issues/8567)) ([bc7007b](https://github.com/windmill-labs/windmill/commit/bc7007bb4265e1f1375c1f0678b74325882a4e92))
+* multi-script dedicated workers race on shared job_dir ([#8551](https://github.com/windmill-labs/windmill/issues/8551)) ([#8569](https://github.com/windmill-labs/windmill/issues/8569)) ([63a3573](https://github.com/windmill-labs/windmill/commit/63a3573951d1f724cc63728ed973d039a5468072))
+* preserve notes on nodes inside collapsed groups ([#8552](https://github.com/windmill-labs/windmill/issues/8552)) ([0fb1153](https://github.com/windmill-labs/windmill/commit/0fb115304afc49812420e9ce24e5048502621059))
+* sanitize flow step summaries for filesystem-safe names ([#8554](https://github.com/windmill-labs/windmill/issues/8554)) ([e15bfbf](https://github.com/windmill-labs/windmill/commit/e15bfbf91ee1517432a6861ebb48e129485006aa))
+* use admin db pool in get_copilot_settings_state ([#8564](https://github.com/windmill-labs/windmill/issues/8564)) ([70f3ee5](https://github.com/windmill-labs/windmill/commit/70f3ee5ed4470e9993be822874f2b38e83a96611))
+
+
+### Performance Improvements
+
+* enable bun bundle caching for WAC v2 scripts ([#8556](https://github.com/windmill-labs/windmill/issues/8556)) ([ab868e9](https://github.com/windmill-labs/windmill/commit/ab868e9ebceadaa55e54770d9d59dc5524da13ff))
+
 ## [1.666.0](https://github.com/windmill-labs/windmill/compare/v1.665.0...v1.666.0) (2026-03-26)
 
 

backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d"
+}

backend/.sqlx/query-1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
+  "query": "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n        ",
   "describe": {
     "columns": [
       {
@@ -50,11 +50,16 @@
       },
       {
         "ordinal": 9,
-        "name": "super_admin",
+        "name": "is_service_account",
         "type_info": "Bool"
       },
       {
         "ordinal": 10,
+        "name": "super_admin!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
         "name": "name",
         "type_info": "Varchar"
       }
@@ -76,8 +81,9 @@
       true,
       true,
       false,
+      null,
       true
     ]
   },
-  "hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
+  "hash": "1cf8597b9d37ec5a924aff8cbc0a05768ed9a679ba908ab16497a9bd55578ba1"
 }

backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            schemaname || '.' || relname as \"table_name!\",\n            pg_total_relation_size(relid) as \"total_size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n        FROM pg_catalog.pg_statio_user_tables\n        ORDER BY pg_total_relation_size(relid) DESC\n        LIMIT 15",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "total_size_bytes!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "total_size_pretty!",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6"
+}

backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2"
+}

backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            ws.workspace_id as \"workspace_id!\",\n            dt.key as \"name!\",\n            dt.value->>'table_name' as \"table_name\"\n        FROM workspace_settings ws,\n             jsonb_each(ws.datatable) dt\n        WHERE dt.value->>'resource_type' = 'instance'\n          AND dt.value->>'table_name' IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "table_name",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null,
+      null
+    ]
+  },
+  "hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720"
+}

backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "size_bytes",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "size_pretty",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null
+    ]
+  },
+  "hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0"
+}

backend/.sqlx/query-544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, is_service_account, disabled FROM usr WHERE username = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_service_account",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "disabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "544a02447bb2cbe8354a5c4ae93685848af38a3461257a9734c43cbd7bd905cb"
+}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -63,7 +68,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "5d6adbe21b9f8dd984d1bfc750fb81763d8650c1316bb0b20816f1a5d61a678c"

backend/.sqlx/query-60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -64,7 +69,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "60b3a59805d463a61eed68072d1ea032b00fc9bd7a6db22f530f67eb9730fa3b"

backend/.sqlx/query-68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND (username = $2 OR email = $3))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "68d1370fa02f4fe585684a91e898c4aed45e6b8f409bb33c2681f92265922040"
+}

backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            c.relname as \"table_name!\",\n            pg_total_relation_size(c.oid) as \"size_bytes!\",\n            pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n            COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n        FROM pg_class c\n        JOIN pg_namespace n ON n.oid = c.relnamespace\n        WHERE n.nspname = 'public' AND c.relname = ANY($1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Name"
+      },
+      {
+        "ordinal": 1,
+        "name": "size_bytes!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "size_pretty!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "estimated_rows!",
+        "type_info": "Float4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "NameArray"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84"
+}

backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "cnt",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9"
+}

backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
   "describe": {
     "columns": [
       {
@@ -12,6 +12,11 @@
         "ordinal": 1,
         "name": "deployment_msg",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamptz"
       }
     ],
     "parameters": {
@@ -22,8 +27,9 @@
     },
     "nullable": [
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de"
+  "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c"
 }

backend/.sqlx/query-9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a"
+}

backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM usr WHERE is_service_account = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf"
+}

backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT AVG(pg_column_size(result))::bigint as \"avg_size\"\n        FROM (\n            SELECT result FROM v2_job_completed\n            WHERE completed_at > now() - interval '30 days'\n              AND result IS NOT NULL\n            ORDER BY completed_at DESC\n            LIMIT $1\n        ) sub",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "avg_size",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00"
+}

backend/.sqlx/query-add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "add01e9e31d64e88b84c9505fe3de553031e581b1bb173413a9a3e3eb0817b43"
+}

backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "oldest",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 1,
+        "name": "total",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null
+    ]
+  },
+  "hash": "b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30"
+}

backend/.sqlx/query-bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3.json

@@ -0,0 +1,44 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            schemaname || '.' || relname as \"table_name!\",\n            COALESCE(n_live_tup, 0) as \"live_tuples!\",\n            COALESCE(n_dead_tup, 0) as \"dead_tuples!\",\n            last_autovacuum as \"last_autovacuum\",\n            last_autoanalyze as \"last_autoanalyze\"\n        FROM pg_stat_user_tables\n        ORDER BY n_dead_tup DESC\n        LIMIT 15",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "table_name!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "live_tuples!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "dead_tuples!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 3,
+        "name": "last_autovacuum",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "last_autoanalyze",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      true,
+      true
+    ]
+  },
+  "hash": "bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3"
+}

backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
   "describe": {
     "columns": [
       {
@@ -12,6 +12,11 @@
         "ordinal": 1,
         "name": "deployment_msg",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamptz"
       }
     ],
     "parameters": {
@@ -22,8 +27,9 @@
     },
     "nullable": [
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec"
+  "hash": "c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462"
 }

backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json

@@ -0,0 +1,46 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            c.id as \"id!\",\n            c.workspace_id as \"workspace_id!\",\n            j.runnable_path as \"runnable_path\",\n            pg_column_size(c.result) as \"result_size_bytes!\",\n            c.completed_at as \"completed_at!\"\n        FROM (\n            SELECT id, workspace_id, result, completed_at\n            FROM v2_job_completed\n            WHERE completed_at > now() - interval '30 days'\n              AND result IS NOT NULL\n            ORDER BY completed_at DESC\n            LIMIT $1\n        ) c\n        LEFT JOIN v2_job j ON j.id = c.id\n        WHERE pg_column_size(c.result) > 1024\n        ORDER BY pg_column_size(c.result) DESC\n        LIMIT 10",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id!",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "runnable_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "result_size_bytes!",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "completed_at!",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      null,
+      false
+    ]
+  },
+  "hash": "dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7"
+}

backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json

@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "added_via",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_service_account",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -63,7 +68,8 @@
       false,
       false,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2"

backend/.sqlx/query-f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token\n            (token_hash, token_prefix, token, email, label, expiration, super_admin, owner)\n            VALUES ($1, $2, $3, $4, $5, $6, false, $7)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f4ad2cf2438c2ae31e388517d09a2c1a2f63ab88cdbc79ffad96c6f9ffb5764b"
+}

backend/.sqlx/query-f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr\n            (workspace_id, email, username, is_admin, operator, is_service_account)\n            VALUES ($1, $2, $3, false, true, true)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f8654d5f50a80d862edbf57355502a9bd039d16f7dfb11e22d16ff9090456853"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.666.0"
+version = "1.668.2"
 authors.workspace = true
 edition.workspace = true
 
@@ -82,7 +82,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.666.0"
+version = "1.668.2"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -260,6 +260,8 @@ windmill-dep-map.workspace = true
 windmill-test-utils.workspace = true
 windmill-worker-volumes.workspace = true
 windmill-types.workspace = true
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 windmill-trigger.workspace = true
 windmill-trigger-websocket.workspace = true
 windmill-trigger-postgres.workspace = true
@@ -362,7 +364,7 @@ reqwest-middleware = { version = "^0", features = ["json"] }
 
 bitflags = "2.9.4"
 memchr = "2.7.4"
-axum = { version = "^0.7", features = ["multipart", "macros"] }
+axum = { version = "^0.8", features = ["multipart", "macros"] }
 headers = "^0"
 hyper = { version = "^1", features = ["full"] }
 hyper-tls = "^0.6"
@@ -371,7 +373,7 @@ tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
 tokio-stream = { version = "0.1.17" }
 tower = "^0"
 tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
-tower-cookies = "^0.10"
+tower-cookies = "^0.11"
 #stuck because of swc for now
 serde = "=1.0.220"
 serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
@@ -386,7 +388,7 @@ tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 tracing-appender = "^0"
 prometheus = { version = "^0", default-features = false }
-cookie = { version = "0.17.0" }
+cookie = { version = "0.18.0" }
 phf = { version = "0.11", features = ["macros"] }
 rust-embed = { version = "^6", features = ["interpolate-folder-path"] }
 mime_guess = "^2"
@@ -415,6 +417,7 @@ time = "^0"
 serde_urlencoded = "^0"
 astral-tokio-tar = "^0.5.6"
 tempfile = "^3"
+x509-parser = "^0.16"
 tokio-util = { version = "=0.7.17", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0.14.0"
@@ -566,18 +569,18 @@ flate2 = "^1"
 http = "^1"
 async-stream = "^0"
 
-opentelemetry = "0.27.0"
-tracing-opentelemetry = "0.28.0"
-opentelemetry_sdk = { version = "0.27.1", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
-opentelemetry-appender-tracing = "0.27.0"
-opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
-opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] }
+opentelemetry = "0.30.0"
+tracing-opentelemetry = "0.31.0"
+opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] }
+opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls"] }
+opentelemetry-appender-tracing = "0.30.0"
+opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] }
+opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] }
 prost = "0.13"
 
 bollard = "0.18.1"
 
-tonic = { version = "=0.12.3", features = ["tls-native-roots"] }
+tonic = { version = "^0.13", features = ["tls-native-roots"] }
 byteorder = "1.5.0"
 
 tikv-jemallocator = { version = "0.5" }

backend/ee-repo-ref.txt

@@ -1 +1 @@
-6db424512b0d02f86489e85f0026581b7637d6e6
+02c0d34e54e71c9293f9cefb56f68652cf0db8a5

backend/migrations/20260326200000_service_accounts.down.sql

@@ -0,0 +1 @@
+ALTER TABLE usr DROP COLUMN is_service_account;

backend/migrations/20260326200000_service_accounts.up.sql

@@ -0,0 +1 @@
+ALTER TABLE usr ADD COLUMN IF NOT EXISTS is_service_account BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20260327000000_email_varchar_255.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(50);
+ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(50);

backend/migrations/20260327000000_email_varchar_255.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE magic_link ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE schedule ALTER COLUMN email TYPE VARCHAR(255);

backend/src/main.rs

@@ -52,9 +52,10 @@ use windmill_common::{
         NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
         OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
         POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
-        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
-        TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING,
+        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
+        RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
+        SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
+        UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING,
     },
     scripts::ScriptLang,
     stats_oss::schedule_stats,
@@ -67,7 +68,7 @@ use windmill_common::{
         is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
         HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
     },
-    KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
+    KillpillSender, DEFAULT_HUB_BASE_URL, INSTANCE_NAME, METRICS_ENABLED,
 };
 
 #[cfg(feature = "enterprise")]
@@ -1791,7 +1792,8 @@ async fn process_notify_event(
                     reload_otel_tracing_proxy_setting(conn).await;
                     if worker_mode {
                         tracing::info!("OTEL tracing proxy setting changed, restarting worker");
-                        send_delayed_killpill(tx, 4, "OTEL tracing proxy setting change").await;
+                        spawn_graceful_killpill(tx, db, 10, "OTEL tracing proxy setting change")
+                            .await;
                     }
                 }
                 REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
@@ -1799,12 +1801,12 @@ async fn process_notify_event(
                 }
                 EXPOSE_METRICS_SETTING => {
                     tracing::info!("Metrics setting changed, restarting");
-                    send_delayed_killpill(tx, 40, "metrics setting change").await;
+                    spawn_graceful_killpill(tx, db, 10, "metrics setting change").await;
                 }
                 EMAIL_DOMAIN_SETTING => {
                     tracing::info!("Email domain setting changed");
                     if server_mode {
-                        send_delayed_killpill(tx, 4, "email domain setting change").await;
+                        spawn_graceful_killpill(tx, db, 10, "email domain setting change").await;
                     }
                 }
                 EXPOSE_DEBUG_METRICS_SETTING => {
@@ -1840,19 +1842,19 @@ async fn process_notify_event(
                 }
                 OTEL_SETTING => {
                     tracing::info!("OTEL setting changed, restarting");
-                    send_delayed_killpill(tx, 4, "OTEL setting change").await;
+                    spawn_graceful_killpill(tx, db, 10, "OTEL setting change").await;
                 }
                 REQUEST_SIZE_LIMIT_SETTING => {
                     if server_mode {
                         tracing::info!("Request limit size change detected, killing server expecting to be restarted");
-                        send_delayed_killpill(tx, 4, "request size limit change").await;
+                        spawn_graceful_killpill(tx, db, 10, "request size limit change").await;
                     }
                 }
                 SAML_METADATA_SETTING => {
                     tracing::info!(
                         "SAML metadata change detected, killing server expecting to be restarted"
                     );
-                    send_delayed_killpill(tx, 0, "SAML metadata change").await;
+                    spawn_graceful_killpill(tx, db, 10, "SAML metadata change").await;
                 }
                 HUB_BASE_URL_SETTING => {
                     if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
@@ -1901,6 +1903,9 @@ async fn process_notify_event(
                     .unwrap_or(false);
                     tracing::info!("Workspace telemetry setting changed: enabled={}", enabled);
                 }
+                RESTART_COORDINATION_SETTING => {
+                    // Internal coordination key for staggered restarts, no action needed
+                }
                 _ => {
                     tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload);
                 }
@@ -2042,14 +2047,145 @@ pub async fn run_workers(
     Ok(())
 }
 
-async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
-    if max_delay_secs == 0 {
-        max_delay_secs = 1;
-    }
-    // Random delay to avoid all servers/workers shutting down simultaneously
-    let rd_delay = rand::rng().random_range(0..max_delay_secs);
-    tracing::info!("Scheduling {context} shutdown in {rd_delay}s");
-    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+/// Schedule a graceful restart with DB-coordinated staggering.
+///
+/// Uses a PostgreSQL advisory lock to serialize restart scheduling across server instances.
+/// Each instance records its planned restart time in the `_restart_coordination` global setting;
+/// subsequent instances read existing schedules and shift their restart to maintain at least
+/// `safety_margin_secs` between consecutive restarts (must exceed the server startup time).
+///
+/// Every server waits at least `DRAIN_DELAY_SECS` to let in-flight requests complete.
+/// Each subsequent server waits an additional `safety_margin_secs` after the previous one,
+/// guaranteeing zero downtime overlap.
+///
+/// The DB coordination is done synchronously (fast, ~ms) to reserve our restart slot,
+/// then the sleep+kill is spawned in the background so the notification handler is not blocked.
+///
+/// Falls back to drain-only delay if DB coordination fails.
+async fn spawn_graceful_killpill(
+    tx: &KillpillSender,
+    db: &Pool<Postgres>,
+    safety_margin_secs: u64,
+    context: &str,
+) {
+    // Minimum delay before any restart to let in-flight requests drain
+    const DRAIN_DELAY_SECS: u64 = 3;
 
-    tx.send();
+    let delay = match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await {
+        Ok(d) => d,
+        Err(e) => {
+            tracing::warn!(
+                "Failed to coordinate restart for {context}: {e:#}, \
+                 falling back to drain delay of {DRAIN_DELAY_SECS}s"
+            );
+            DRAIN_DELAY_SECS
+        }
+    };
+
+    tracing::info!("Scheduling {context} graceful shutdown in {delay}s");
+    let tx = tx.clone();
+    tokio::spawn(async move {
+        tokio::time::sleep(Duration::from_secs(delay)).await;
+        tx.send();
+    });
+}
+
+/// Coordinate a restart delay with other instances via the DB.
+///
+/// Returns the delay (in seconds from now) at which this instance should restart.
+/// The first server gets `drain_delay_secs` (to let in-flight requests complete).
+/// Each subsequent server is spaced `safety_margin_secs` after the latest scheduled restart.
+async fn coordinate_restart_delay(
+    db: &Pool<Postgres>,
+    safety_margin_secs: u64,
+    drain_delay_secs: u64,
+) -> anyhow::Result<u64> {
+    const RESTART_LOCK_ID: i64 = 737_483_920;
+    // Stale threshold: ignore coordination entries older than this
+    const STALE_THRESHOLD_SECS: i64 = 120;
+
+    let now = chrono::Utc::now();
+
+    let mut tx = db.begin().await.context("begin restart coordination tx")?;
+
+    // Serialize access across all instances
+    sqlx::query("SELECT pg_advisory_xact_lock($1)")
+        .bind(RESTART_LOCK_ID)
+        .execute(&mut *tx)
+        .await
+        .context("acquire restart coordination lock")?;
+
+    // Read existing coordination record
+    let existing: Option<serde_json::Value> =
+        sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1")
+            .bind(RESTART_COORDINATION_SETTING)
+            .fetch_optional(&mut *tx)
+            .await
+            .context("read restart coordination")?;
+
+    // Parse existing scheduled restarts, filtering out stale entries
+    // Each entry is (instance_name, restart_at)
+    let mut scheduled: Vec<(String, chrono::DateTime<chrono::Utc>)> = Vec::new();
+    if let Some(val) = &existing {
+        if let Some(arr) = val.get("restarts").and_then(|v| v.as_array()) {
+            for entry in arr {
+                let instance = entry
+                    .get("instance")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("unknown")
+                    .to_string();
+                if let Some(ts_str) = entry.get("restart_at").and_then(|v| v.as_str()) {
+                    if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(ts_str) {
+                        let dt = dt.with_timezone(&chrono::Utc);
+                        let stale_cutoff = now - chrono::Duration::seconds(STALE_THRESHOLD_SECS);
+                        if dt > stale_cutoff {
+                            scheduled.push((instance, dt));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Find the latest scheduled restart
+    let latest = scheduled.iter().map(|(_, dt)| *dt).max();
+    let earliest_allowed = now + chrono::Duration::seconds(drain_delay_secs as i64);
+
+    // Our restart time: drain_delay from now, or safety_margin after the latest existing restart
+    let our_restart = match latest {
+        Some(last) => {
+            let after_last = last + chrono::Duration::seconds(safety_margin_secs as i64);
+            // Use whichever is later: drain delay or staggered position
+            earliest_allowed.max(after_last)
+        }
+        None => earliest_allowed,
+    };
+
+    // Record our restart time (deduplicate: remove any prior entry for this instance)
+    scheduled.retain(|(inst, _)| inst != &*INSTANCE_NAME);
+    scheduled.push((INSTANCE_NAME.clone(), our_restart));
+    let new_value = serde_json::json!({
+        "restarts": scheduled.iter().map(|(inst, dt)| {
+            serde_json::json!({
+                "instance": inst,
+                "restart_at": dt.to_rfc3339()
+            })
+        }).collect::<Vec<_>>()
+    });
+
+    sqlx::query(
+        "INSERT INTO global_settings (name, value, updated_at) \
+         VALUES ($1, $2, now()) \
+         ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+    )
+    .bind(RESTART_COORDINATION_SETTING)
+    .bind(&new_value)
+    .execute(&mut *tx)
+    .await
+    .context("write restart coordination")?;
+
+    tx.commit().await.context("commit restart coordination")?;
+
+    let delay = (our_restart - now).num_seconds().max(0) as u64;
+    Ok(delay)
 }

backend/src/monitor.rs

@@ -169,6 +169,8 @@ lazy_static::lazy_static! {
 
     static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref QUEUE_RUNNING_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+    static ref OTEL_QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
+    static ref OTEL_QUEUE_RUNNING_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
 
     //legacy typo
@@ -2372,8 +2374,20 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             }
         }
 
+        let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed);
+
+        if otel_enabled {
+            for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() {
+                if queue_counts.get(q).is_none() {
+                    otel_set_queue_count(q, 0);
+                }
+            }
+        }
+
         #[allow(unused_mut)]
         let mut tags_to_watch = vec![];
+        #[allow(unused_mut)]
+        let mut otel_tags_to_watch = vec![];
         for q in queue_counts {
             let count = q.1;
             let tag = q.0;
@@ -2385,6 +2399,9 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 tags_to_watch.push(tag.to_string());
             }
 
+            if otel_enabled {
+                otel_tags_to_watch.push(tag.to_string());
+            }
             otel_set_queue_count(&tag, count as i64);
 
             // save queue_count and delay metrics per tag
@@ -2419,9 +2436,13 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             let mut w = QUEUE_COUNT_TAGS.write().await;
             *w = tags_to_watch;
         }
+        if otel_enabled {
+            let mut w = OTEL_QUEUE_COUNT_TAGS.write().await;
+            *w = otel_tags_to_watch;
+        }
 
         // Single DB query for running counts, shared by Prometheus and OTel
-        let otel_running = OTEL_METRICS_ENABLED.load(Ordering::Relaxed);
+        let otel_running = otel_enabled;
         #[cfg(feature = "prometheus")]
         let need_running_counts = metrics_enabled || otel_running;
         #[cfg(not(feature = "prometheus"))]
@@ -2439,8 +2460,18 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 }
             }
 
+            if otel_running {
+                for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() {
+                    if queue_running_counts.get(q).is_none() {
+                        otel_set_queue_running_count(q, 0);
+                    }
+                }
+            }
+
             #[allow(unused_mut, unused_variables)]
             let mut running_tags_to_watch: Vec<String> = vec![];
+            #[allow(unused_mut, unused_variables)]
+            let mut otel_running_tags_to_watch: Vec<String> = vec![];
             for (tag, count) in &queue_running_counts {
                 #[cfg(feature = "prometheus")]
                 if metrics_enabled {
@@ -2451,6 +2482,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
 
                 if otel_running {
                     otel_set_queue_running_count(tag, *count as i64);
+                    otel_running_tags_to_watch.push(tag.to_string());
                 }
             }
 
@@ -2459,6 +2491,10 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
                 let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await;
                 *w = running_tags_to_watch;
             }
+            if otel_running {
+                let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await;
+                *w = otel_running_tags_to_watch;
+            }
         }
     }
 

backend/tests/fixtures/typechecked_python.sql

@@ -0,0 +1,20 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import inspect
+import sys
+
+def greet(name: str) -> str:
+    # Verify that __file__ is set on this module (same check typeguard does)
+    mod = sys.modules[__name__]
+    source_file = inspect.getfile(mod)
+    return f"Hello, {name}! from {source_file}"
+
+def main():
+    return greet("World")
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/typechecked_helper', 12349, 'python3', '');

backend/tests/otel.rs

@@ -0,0 +1,504 @@
+//! E2E tests for OpenTelemetry integration.
+//!
+//! Verify that metrics are recorded with correct names/values/attributes and
+//! spans are created with correct trace IDs, attributes, and status codes.
+//!
+//! Run with: cargo test --features enterprise,private,otel --test otel -- --test-threads=1
+
+#![cfg(all(feature = "otel", feature = "enterprise"))]
+
+use std::sync::{atomic::Ordering, Arc};
+
+use opentelemetry::global;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::{
+    metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider},
+    trace::{InMemorySpanExporter, SdkTracerProvider, SimpleSpanProcessor},
+};
+use windmill_common::otel_ee::*;
+use windmill_common::{OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED};
+
+// ── Global test infrastructure ──────────────────────────────────────────
+
+struct OtelTestState {
+    metric_exporter: InMemoryMetricExporter,
+    span_exporter: InMemorySpanExporter,
+    meter_provider: SdkMeterProvider,
+}
+
+static STATE: tokio::sync::OnceCell<Arc<OtelTestState>> = tokio::sync::OnceCell::const_new();
+
+async fn ensure_setup() -> Arc<OtelTestState> {
+    STATE
+        .get_or_init(|| async {
+            // Metrics: InMemoryMetricExporter + PeriodicReader (needs async tokio context)
+            let metric_exporter = InMemoryMetricExporter::default();
+            let reader = PeriodicReader::builder(metric_exporter.clone()).build();
+            let meter_provider = SdkMeterProvider::builder().with_reader(reader).build();
+            global::set_meter_provider(meter_provider.clone());
+            OTEL_METRICS_ENABLED.store(true, Ordering::SeqCst);
+
+            // Tracing: InMemorySpanExporter + SimpleSpanProcessor
+            let span_exporter = InMemorySpanExporter::default();
+            let tracer_provider = SdkTracerProvider::builder()
+                .with_span_processor(SimpleSpanProcessor::new(span_exporter.clone()))
+                .build();
+            let tracer = tracer_provider.tracer("windmill");
+            *TRACER.write().unwrap() = Some(tracer);
+            OTEL_TRACING_ENABLED.store(true, Ordering::SeqCst);
+
+            Arc::new(OtelTestState { metric_exporter, span_exporter, meter_provider })
+        })
+        .await
+        .clone()
+}
+
+// ── Metric helper: flush + collect ──────────────────────────────────────
+
+fn flush_and_get_metrics(
+    state: &OtelTestState,
+) -> Vec<opentelemetry_sdk::metrics::data::ResourceMetrics> {
+    state.meter_provider.force_flush().expect("flush failed");
+    state
+        .metric_exporter
+        .get_finished_metrics()
+        .expect("get_finished_metrics failed")
+}
+
+fn find_metric<'a>(
+    all: &'a [opentelemetry_sdk::metrics::data::ResourceMetrics],
+    name: &str,
+) -> Option<&'a opentelemetry_sdk::metrics::data::Metric> {
+    all.iter()
+        .flat_map(|rm| rm.scope_metrics())
+        .flat_map(|sm| sm.metrics())
+        .find(|m| m.name() == name)
+}
+
+fn metric_names(all: &[opentelemetry_sdk::metrics::data::ResourceMetrics]) -> Vec<String> {
+    all.iter()
+        .flat_map(|rm| rm.scope_metrics())
+        .flat_map(|sm| sm.metrics())
+        .map(|m| m.name().to_string())
+        .collect()
+}
+
+// ── Counter value helpers ───────────────────────────────────────────────
+
+fn sum_u64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<u64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::U64(MetricData::Sum(sum)) => {
+            Some(sum.data_points().map(|dp| dp.value()).sum())
+        }
+        _ => None,
+    }
+}
+
+fn gauge_i64_values(
+    metric: &opentelemetry_sdk::metrics::data::Metric,
+) -> Vec<(Vec<opentelemetry::KeyValue>, i64)> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::I64(MetricData::Gauge(gauge)) => gauge
+            .data_points()
+            .map(|dp| (dp.attributes().cloned().collect(), dp.value()))
+            .collect(),
+        _ => panic!("expected I64 Gauge metric"),
+    }
+}
+
+fn gauge_f64_value(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<f64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Gauge(gauge)) => {
+            gauge.data_points().next().map(|dp| dp.value())
+        }
+        _ => None,
+    }
+}
+
+fn histogram_f64_count(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<u64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Histogram(hist)) => {
+            Some(hist.data_points().map(|dp| dp.count()).sum())
+        }
+        _ => None,
+    }
+}
+
+fn histogram_f64_sum(metric: &opentelemetry_sdk::metrics::data::Metric) -> Option<f64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    match metric.data() {
+        AggregatedMetrics::F64(MetricData::Histogram(hist)) => {
+            Some(hist.data_points().map(|dp| dp.sum()).sum())
+        }
+        _ => None,
+    }
+}
+
+// ═══════════════════════════════════════════════════════════════════════
+// METRICS E2E TEST
+//
+// All metric assertions live in one test function because the PeriodicReader's
+// background task is tied to the tokio runtime that created it. Separate
+// #[tokio::test] functions each get their own runtime, and the reader becomes
+// disconnected after the first test's runtime is dropped.
+// ═══════════════════════════════════════════════════════════════════════
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_all_metrics_e2e() {
+    let state = ensure_setup().await;
+
+    // ── Counters ────────────────────────────────────────────────────
+
+    otel_incr_queue_push_count();
+    otel_incr_queue_push_count();
+    otel_incr_queue_push_count();
+    otel_incr_queue_delete_count();
+    otel_incr_queue_pull_count();
+    otel_incr_zombie_restart_count(7);
+    otel_incr_zombie_delete_count(3);
+    otel_incr_worker_execution_count("bun");
+    otel_incr_worker_execution_count("bun");
+    otel_incr_worker_execution_failed("go");
+    otel_incr_worker_started();
+
+    // ── Gauges ──────────────────────────────────────────────────────
+
+    otel_set_queue_count("python3", 42);
+    otel_set_queue_running_count("deno", 5);
+    otel_set_worker_busy("worker-test-1", 1);
+    otel_set_db_pool(5, 10, 20);
+    otel_set_health_db_latency(2.5);
+    otel_set_worker_uptime("w-uptime", 3600.0);
+    otel_set_health_status_phase("healthy");
+    otel_set_health_db_unresponsive(true);
+
+    // ── Histograms ──────────────────────────────────────────────────
+
+    otel_record_worker_execution_duration("python3", 1.5);
+    otel_record_worker_execution_duration("python3", 2.5);
+    otel_record_worker_pull_duration("w1", true, 0.05);
+    otel_record_worker_pull_duration("w1", false, 0.01);
+
+    // ── Flush and collect ───────────────────────────────────────────
+
+    let metrics = flush_and_get_metrics(&state);
+    let names = metric_names(&metrics);
+
+    // ── Verify all 20 metric names are present ──────────────────────
+
+    let expected = [
+        "windmill.queue.push_count",
+        "windmill.queue.delete_count",
+        "windmill.queue.pull_count",
+        "windmill.queue.zombie_restart_count",
+        "windmill.queue.zombie_delete_count",
+        "windmill.queue.count",
+        "windmill.queue.running_count",
+        "windmill.worker.execution_count",
+        "windmill.worker.execution_duration",
+        "windmill.worker.busy",
+        "windmill.worker.pull_duration",
+        "windmill.worker.execution_failed",
+        "windmill.db.pool.active",
+        "windmill.db.pool.idle",
+        "windmill.db.pool.max",
+        "windmill.health.db_latency",
+        "windmill.worker.started",
+        "windmill.worker.uptime",
+        "windmill.health.status",
+        "windmill.health.db_unresponsive",
+    ];
+    for name in expected {
+        assert!(
+            names.iter().any(|n| n == name),
+            "metric '{}' not found in {:?}",
+            name,
+            names
+        );
+    }
+
+    // ── Counter values ──────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.queue.push_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 3, "push_count should be >= 3");
+
+    let m = find_metric(&metrics, "windmill.queue.delete_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.queue.pull_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.queue.zombie_restart_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 7);
+
+    let m = find_metric(&metrics, "windmill.queue.zombie_delete_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 3);
+
+    let m = find_metric(&metrics, "windmill.worker.execution_count").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 2);
+
+    let m = find_metric(&metrics, "windmill.worker.execution_failed").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    let m = find_metric(&metrics, "windmill.worker.started").unwrap();
+    assert!(sum_u64_value(m).unwrap() >= 1);
+
+    // ── Gauge values ────────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.queue.count").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "python3")
+        })
+        .expect("queue.count data point with tag=python3 not found");
+    assert_eq!(dp.1, 42);
+
+    let m = find_metric(&metrics, "windmill.queue.running_count").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "tag" && kv.value.as_str() == "deno")
+        })
+        .expect("running_count data point with tag=deno not found");
+    assert_eq!(dp.1, 5);
+
+    let m = find_metric(&metrics, "windmill.worker.busy").unwrap();
+    let values = gauge_i64_values(m);
+    let dp = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "worker" && kv.value.as_str() == "worker-test-1")
+        })
+        .expect("worker.busy data point with worker=worker-test-1 not found");
+    assert_eq!(dp.1, 1);
+
+    let m = find_metric(&metrics, "windmill.db.pool.active").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 5);
+    let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 10);
+    let m = find_metric(&metrics, "windmill.db.pool.max").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 20);
+
+    let m = find_metric(&metrics, "windmill.health.db_latency").unwrap();
+    assert!((gauge_f64_value(m).unwrap() - 2.5).abs() < f64::EPSILON);
+
+    let m = find_metric(&metrics, "windmill.worker.uptime").unwrap();
+    assert!((gauge_f64_value(m).unwrap() - 3600.0).abs() < f64::EPSILON);
+
+    let m = find_metric(&metrics, "windmill.health.db_unresponsive").unwrap();
+    assert_eq!(gauge_i64_values(m)[0].1, 1);
+
+    // ── Health status phase (all 3 phases) ──────────────────────────
+
+    let m = find_metric(&metrics, "windmill.health.status").unwrap();
+    let values = gauge_i64_values(m);
+    let healthy = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "healthy")
+        })
+        .expect("phase=healthy");
+    let degraded = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "degraded")
+        })
+        .expect("phase=degraded");
+    let unhealthy = values
+        .iter()
+        .find(|(attrs, _)| {
+            attrs
+                .iter()
+                .any(|kv| kv.key.as_str() == "phase" && kv.value.as_str() == "unhealthy")
+        })
+        .expect("phase=unhealthy");
+    assert_eq!(healthy.1, 1);
+    assert_eq!(degraded.1, 0);
+    assert_eq!(unhealthy.1, 0);
+
+    // ── Histogram values ────────────────────────────────────────────
+
+    let m = find_metric(&metrics, "windmill.worker.execution_duration").unwrap();
+    assert!(histogram_f64_count(m).unwrap() >= 2);
+    assert!(histogram_f64_sum(m).unwrap() >= 4.0);
+
+    let m = find_metric(&metrics, "windmill.worker.pull_duration").unwrap();
+    assert!(histogram_f64_count(m).unwrap() >= 2);
+}
+
+// ═══════════════════════════════════════════════════════════════════════
+// SPAN E2E TESTS
+// ═══════════════════════════════════════════════════════════════════════
+
+fn make_test_job(id: uuid::Uuid, parent: Option<uuid::Uuid>) -> windmill_queue::MiniPulledJob {
+    use windmill_types::jobs::JobKind;
+    let mut job = windmill_queue::MiniPulledJob::new_inline(
+        "test-workspace".to_string(),
+        None,
+        "test-user".to_string(),
+        "u/test-user".to_string(),
+        "test@example.com".to_string(),
+        Some("f/test/script".to_string()),
+        JobKind::Script,
+        None,
+        "deno".to_string(),
+        None,
+    );
+    job.id = id;
+    job.parent_job = parent;
+    job.started_at = Some(chrono::Utc::now());
+    job
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_created_on_success() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    assert_eq!(span.status, opentelemetry::trace::Status::Ok,);
+
+    // Verify attributes
+    let attrs: Vec<_> = span.attributes.iter().map(|kv| kv.key.as_str()).collect();
+    assert!(attrs.contains(&"job_id"), "missing job_id attribute");
+    assert!(
+        attrs.contains(&"workspace_id"),
+        "missing workspace_id attribute"
+    );
+    assert!(
+        attrs.contains(&"script_path"),
+        "missing script_path attribute"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_error_on_failure() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, false);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    match &span.status {
+        opentelemetry::trace::Status::Error { description } => {
+            assert_eq!(description.as_ref(), "Job failed");
+        }
+        other => panic!("expected Error status, got {:?}", other),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_trace_id_matches_uuid() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let expected_trace_id =
+        opentelemetry::trace::TraceId::from_bytes(job_id.as_u128().to_be_bytes());
+    assert_eq!(span.span_context.trace_id(), expected_trace_id);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_id_matches_uuid() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let expected_span_id =
+        opentelemetry::trace::SpanId::from_bytes(job_id.as_u64_pair().1.to_be_bytes());
+    assert_eq!(span.span_context.span_id(), expected_span_id);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_child_job_produces_no_span() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let parent_id = uuid::Uuid::new_v4();
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, Some(parent_id));
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let found = spans.iter().any(|s| s.name == "full_job");
+    assert!(!found, "child job should not produce a span");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_root_job_span_attributes_values() {
+    let state = ensure_setup().await;
+    state.span_exporter.reset();
+
+    let job_id = uuid::Uuid::new_v4();
+    let job = make_test_job(job_id, None);
+    windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
+
+    let spans = state.span_exporter.get_finished_spans().unwrap();
+    let span = spans
+        .iter()
+        .find(|s| s.name == "full_job")
+        .expect("full_job span not found");
+
+    let get_attr = |key: &str| -> String {
+        span.attributes
+            .iter()
+            .find(|kv| kv.key.as_str() == key)
+            .map(|kv| kv.value.as_str().to_string())
+            .unwrap_or_default()
+    };
+
+    assert_eq!(get_attr("job_id"), job_id.to_string());
+    assert_eq!(get_attr("workspace_id"), "test-workspace");
+    assert_eq!(get_attr("script_path"), "f/test/script");
+}

backend/tests/python_jobs.rs

@@ -923,3 +923,43 @@ async def main(item: str, qty: int, email: str):
     .await;
     Ok(())
 }
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "typechecked_python"))]
+async fn test_typechecked_decorator_python(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let content = r#"
+from f.system.typechecked_helper import greet
+
+def main():
+    return greet("World")
+"#
+    .to_owned();
+
+    let job = JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: Some("f/system/test_typechecked".to_string()),
+        language: ScriptLang::Python3,
+        lock: None,
+        concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
+            .into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        modules: None,
+    });
+
+    let result = run_job_in_new_worker_until_complete(&db, false, job, port)
+        .await
+        .json_result()
+        .unwrap();
+
+    let result_str = result.as_str().unwrap();
+    assert!(result_str.starts_with("Hello, World! from "), "unexpected result: {result_str}");
+    Ok(())
+}

backend/windmill-api-auth/src/auth.rs

@@ -1,7 +1,6 @@
 #[cfg(feature = "enterprise")]
 use crate::ee_oss::ExternalJwks;
 use axum::{
-    async_trait,
     extract::{FromRequestParts, OriginalUri, Query},
     Extension, Json,
 };
@@ -226,7 +225,15 @@ impl AuthCache {
                     t_hash,
                     w_id.as_ref(),
                 )
-                .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
+                .map(|x| {
+                    (
+                        x.owner,
+                        x.email,
+                        x.super_admin,
+                        x.scopes,
+                        x.label,
+                    )
+                })
                 .fetch_optional(&self.db)
                 .await
                 .ok()
@@ -235,7 +242,13 @@ impl AuthCache {
                 if let Some(user) = user_o {
                     let authed_o = {
                         match user {
-                            (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
+                            (
+                                Some(owner),
+                                Some(email),
+                                super_admin,
+                                _,
+                                label,
+                            ) if w_id.is_some() => {
                                 let username_override = username_override_from_label(label);
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
@@ -451,7 +464,11 @@ pub(crate) async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S)
         None => Extension::<Cookies>::from_request_parts(parts, state)
             .await
             .ok()
-            .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
+            .and_then(|cookies| {
+                cookies
+                    .get(COOKIE_NAME)
+                    .map(|c| c.value_trimmed().to_owned())
+            }),
     };
 
     #[derive(Deserialize)]
@@ -504,7 +521,6 @@ impl BruteForceCounter {
     }
 }
 
-#[async_trait]
 impl<S> FromRequestParts<S> for Tokened
 where
     S: Send + Sync,
@@ -535,7 +551,6 @@ where
     }
 }
 
-#[async_trait]
 impl<S> FromRequestParts<S> for OptTokened
 where
     S: Send + Sync,

backend/windmill-api-auth/src/lib.rs

@@ -12,8 +12,7 @@ pub mod ee;
 pub mod ee_oss;
 pub mod scopes;
 
-use axum::async_trait;
-use axum::extract::FromRequestParts;
+use axum::extract::{FromRequestParts, OptionalFromRequestParts};
 use http::request::Parts;
 
 use windmill_audit::audit_oss::AuditAuthorable;
@@ -345,7 +344,6 @@ pub async fn maybe_refresh_folders(
 
 // ------------ FromRequestParts impls (direct call to auth module) ------------
 
-#[async_trait]
 impl<S> FromRequestParts<S> for ApiAuthed
 where
     S: Send + Sync,
@@ -361,7 +359,24 @@ where
     }
 }
 
-#[async_trait]
+impl<S> OptionalFromRequestParts<S> for ApiAuthed
+where
+    S: Send + Sync,
+{
+    type Rejection = std::convert::Infallible;
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &S,
+    ) -> std::result::Result<Option<Self>, Self::Rejection> {
+        Ok(
+            <Self as FromRequestParts<S>>::from_request_parts(parts, state)
+                .await
+                .ok(),
+        )
+    }
+}
+
 impl<S> FromRequestParts<S> for OptJobAuthed
 where
     S: Send + Sync,
@@ -397,7 +412,6 @@ fn empty_parts() -> Parts {
 #[derive(Clone, Debug)]
 pub struct OptAuthed(pub Option<ApiAuthed>);
 
-#[async_trait]
 impl<S> FromRequestParts<S> for OptAuthed
 where
     S: Send + Sync,
@@ -408,7 +422,7 @@ where
         parts: &mut Parts,
         state: &S,
     ) -> std::result::Result<Self, Self::Rejection> {
-        ApiAuthed::from_request_parts(parts, state)
+        <ApiAuthed as FromRequestParts<S>>::from_request_parts(parts, state)
             .await
             .map(|authed| Self(Some(authed)))
             .or_else(|_| Ok(Self(None)))

backend/windmill-api-configs/src/lib.rs

@@ -28,11 +28,11 @@ use windmill_api_auth::{require_devops_role, ApiAuthed};
 pub fn global_service() -> Router {
     Router::new()
         .route("/list_worker_groups", get(list_worker_groups))
-        .route("/update/:name", post(update_config).delete(delete_config))
-        .route("/get/:name", get(get_config))
+        .route("/update/{name}", post(update_config).delete(delete_config))
+        .route("/get/{name}", get(get_config))
         .route("/list", get(list_configs))
         .route(
-            "/list_autoscaling_events/:worker_group",
+            "/list_autoscaling_events/{worker_group}",
             get(list_autoscaling_events),
         )
         .route(

backend/windmill-api-flow-conversations/src/lib.rs

@@ -21,8 +21,8 @@ use windmill_common::{
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_conversations))
-        .route("/delete/:conversation_id", delete(delete_conversation))
-        .route("/:conversation_id/messages", get(list_messages))
+        .route("/delete/{conversation_id}", delete(delete_conversation))
+        .route("/{conversation_id}/messages", get(list_messages))
 }
 
 #[derive(Serialize, FromRow, Debug)]

backend/windmill-api-flows/src/flows.rs

@@ -61,26 +61,26 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_flows))
         .route("/list_search", get(list_search_flows))
         .route("/create", post(create_flow))
-        .route("/update/*path", post(update_flow))
-        .route("/archive/*path", post(archive_flow_by_path))
-        .route("/delete/*path", delete(delete_flow_by_path))
-        .route("/list_tokens/*path", get(list_tokens))
-        .route("/get/*path", get(get_flow_by_path))
-        .route("/deployment_status/p/*path", get(get_deployment_status))
-        .route("/get/draft/*path", get(get_flow_by_path_w_draft))
-        .route("/exists/*path", get(exists_flow_by_path))
+        .route("/update/{*path}", post(update_flow))
+        .route("/archive/{*path}", post(archive_flow_by_path))
+        .route("/delete/{*path}", delete(delete_flow_by_path))
+        .route("/list_tokens/{*path}", get(list_tokens))
+        .route("/get/{*path}", get(get_flow_by_path))
+        .route("/deployment_status/p/{*path}", get(get_deployment_status))
+        .route("/get/draft/{*path}", get(get_flow_by_path_w_draft))
+        .route("/exists/{*path}", get(exists_flow_by_path))
         .route("/list_paths", get(list_paths))
-        .route("/history/p/*path", get(get_flow_history))
-        .route("/get_latest_version/*path", get(get_latest_version))
+        .route("/history/p/{*path}", get(get_flow_history))
+        .route("/get_latest_version/{*path}", get(get_latest_version))
         .route(
-            "/list_paths_from_workspace_runnable/:runnable_kind/*path",
+            "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}",
             get(list_paths_from_workspace_runnable),
         )
-        .route("/history_update/v/:version", post(update_flow_history))
-        .route("/get/v/:version", get(get_flow_version_by_id))
-        .route("/get/v/:version/p/*path", get(get_flow_version))
+        .route("/history_update/v/{version}", post(update_flow_history))
+        .route("/get/v/{version}", get(get_flow_version_by_id))
+        .route("/get/v/{version}/p/{*path}", get(get_flow_version))
         .route(
-            "/toggle_workspace_error_handler/*path",
+            "/toggle_workspace_error_handler/{*path}",
             post(toggle_workspace_error_handler),
         )
 }
@@ -88,7 +88,7 @@ pub fn workspaced_service() -> Router {
 pub fn global_service() -> Router {
     Router::new()
         .route("/hub/list", get(list_hub_flows))
-        .route("/hub/get/:id", get(get_hub_flow_by_id))
+        .route("/hub/get/{id}", get(get_hub_flow_by_id))
 }
 
 #[derive(Serialize, FromRow)]

backend/windmill-api-groups/src/folder_history.rs

@@ -22,7 +22,7 @@ use serde::Serialize;
 use sqlx::FromRow;
 
 pub fn workspaced_service() -> Router {
-    Router::new().route("/get/:name", get(get_folder_permission_history))
+    Router::new().route("/get/{name}", get(get_folder_permission_history))
 }
 
 #[derive(Serialize, FromRow)]

backend/windmill-api-groups/src/folders.rs

@@ -40,14 +40,14 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_folders))
         .route("/listnames", get(list_foldernames))
         .route("/create", post(create_folder))
-        .route("/get/:name", get(get_folder))
-        .route("/exists/:name", get(exists_folder))
-        .route("/update/:name", post(update_folder))
-        .route("/getusage/:name", get(get_folder_usage))
-        .route("/delete/:name", delete(delete_folder))
-        .route("/addowner/:name", post(add_owner))
-        .route("/removeowner/:name", post(remove_owner))
-        .route("/is_owner/*path", get(is_owner_api))
+        .route("/get/{name}", get(get_folder))
+        .route("/exists/{name}", get(exists_folder))
+        .route("/update/{name}", post(update_folder))
+        .route("/getusage/{name}", get(get_folder_usage))
+        .route("/delete/{name}", delete(delete_folder))
+        .route("/addowner/{name}", post(add_owner))
+        .route("/removeowner/{name}", post(remove_owner))
+        .route("/is_owner/{*path}", get(is_owner_api))
 }
 
 #[derive(FromRow, Serialize, Deserialize, Clone)]

backend/windmill-api-groups/src/granular_acls.rs

@@ -48,9 +48,9 @@ const KINDS: [&str; 19] = [
 
 pub fn workspaced_service() -> Router {
     Router::new()
-        .route("/get/*path", get(get_granular_acls))
-        .route("/add/*path", post(add_granular_acl))
-        .route("/remove/*path", post(remove_granular_acl))
+        .route("/get/{*path}", get(get_granular_acls))
+        .route("/add/{*path}", post(add_granular_acl))
+        .route("/remove/{*path}", post(remove_granular_acl))
 }
 
 #[derive(Serialize, Deserialize)]

backend/windmill-api-groups/src/groups.rs

@@ -33,24 +33,24 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_groups))
         .route("/listnames", get(list_group_names))
         .route("/create", post(create_group))
-        .route("/get/:name", get(get_group))
-        .route("/update/:name", post(update_group))
-        .route("/delete/:name", delete(delete_group))
-        .route("/adduser/:name", post(add_user))
-        .route("/removeuser/:name", post(remove_user))
-        .route("/is_owner/:name", get(is_owner))
+        .route("/get/{name}", get(get_group))
+        .route("/update/{name}", post(update_group))
+        .route("/delete/{name}", delete(delete_group))
+        .route("/adduser/{name}", post(add_user))
+        .route("/removeuser/{name}", post(remove_user))
+        .route("/is_owner/{name}", get(is_owner))
 }
 
 pub fn global_service() -> Router {
     Router::new()
         .route("/list", get(list_igroups))
         .route("/list_with_workspaces", get(list_igroups_with_workspaces))
-        .route("/get/:name", get(get_igroup))
+        .route("/get/{name}", get(get_igroup))
         .route("/create", post(create_igroup))
-        .route("/update/:name", post(update_igroup))
-        .route("/delete/:name", delete(delete_igroup))
-        .route("/adduser/:name", post(add_user_igroup))
-        .route("/removeuser/:name", post(remove_user_igroup))
+        .route("/update/{name}", post(update_igroup))
+        .route("/delete/{name}", delete(delete_igroup))
+        .route("/adduser/{name}", post(add_user_igroup))
+        .route("/removeuser/{name}", post(remove_user_igroup))
         .route("/export", get(export_igroups))
         .route("/overwrite", post(overwrite_igroups))
 }

backend/windmill-api-inputs/src/lib.rs

@@ -33,9 +33,9 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_saved_inputs))
         .route("/create", post(create_input))
         .route("/update", post(update_input))
-        .route("/delete/:id", post(delete_input))
+        .route("/delete/{id}", post(delete_input))
         .route(
-            "/:job_or_input_id/args",
+            "/{job_or_input_id}/args",
             get(get_args_from_history_or_saved_input),
         )
 }

backend/windmill-api-integration-tests/Cargo.toml

@@ -40,3 +40,4 @@ aws-config = { workspace = true, optional = true }
 aws-credential-types = { workspace = true, optional = true }
 aws-sdk-sqs = { workspace = true, optional = true }
 base64 = { workspace = true, optional = true }
+axum.workspace = true

backend/windmill-api-integration-tests/tests/ai_routes.rs

@@ -0,0 +1,106 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+/// Start a mock AI API that echoes back a valid chat completion response.
+async fn start_mock_ai_api() -> u16 {
+    use axum::{routing::post, Json, Router};
+
+    let app = Router::new().fallback(post(|| async {
+        Json(json!({
+            "id": "chatcmpl-test",
+            "object": "chat.completion",
+            "choices": [{"message": {"role": "assistant", "content": "hello"}}]
+        }))
+    }));
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let port = listener.local_addr().unwrap().port();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    port
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_ai_proxy_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // Start mock AI API
+    let mock_port = start_mock_ai_api().await;
+    let mock_url = format!("http://127.0.0.1:{mock_port}/v1");
+
+    // Create an openai resource pointing to the mock
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/resources/create"
+            ))
+            .json(&json!({
+                "path": "f/ai/openai_config",
+                "resource_type": "openai",
+                "value": {
+                    "api_key": "test-key",
+                    "base_url": mock_url
+                }
+            })),
+    )
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "create openai resource",
+    );
+
+    // Set ai_config on workspace_settings directly via SQL
+    sqlx::query(
+        "UPDATE workspace_settings SET ai_config = $1::jsonb WHERE workspace_id = 'test-workspace'",
+    )
+    .bind(json!({
+        "providers": {
+            "openai": {
+                "resource_path": "f/ai/openai_config",
+                "models": ["gpt-4"]
+            }
+        }
+    }))
+    .execute(&db)
+    .await?;
+
+    // POST /w/{ws}/ai/proxy/chat/completions
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/ai/proxy/chat/completions"
+            ))
+            .header("X-Provider", "openai")
+            .json(&json!({
+                "model": "gpt-4",
+                "messages": [{"role": "user", "content": "hi"}]
+            })),
+    )
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /ai/proxy/chat/completions",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/audit.rs

@@ -0,0 +1,35 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_audit_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/audit");
+
+    // GET /list returns 200 (empty array)
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /audit/list",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/capture_unauthed.rs

@@ -0,0 +1,83 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_capture_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // POST /capture/set_config → 200 (authed)
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/capture/set_config"
+            ))
+            .json(&json!({
+                "trigger_kind": "webhook",
+                "path": "u/test-user/test_capture",
+                "is_flow": false
+            })),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /capture/set_config");
+
+    // GET /capture/list/{...} → 200 (authed)
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/list/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(
+        status,
+        &body,
+        "GET /capture/list/script/u/test-user/test_capture",
+    );
+
+    // POST /capture/ping_config/{trigger_kind}/{runnable_kind}/{*path} → 200
+    let resp = authed(client().post(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/ping_config/webhook/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /capture/ping_config",
+    );
+
+    // GET /capture/get_configs/{runnable_kind}/{*path} → 200
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/capture/get_configs/script/u/test-user/test_capture"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /capture/get_configs",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/concurrency_groups.rs

@@ -0,0 +1,48 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_concurrency_groups_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/concurrency_groups/list"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /api/concurrency_groups/list",
+    );
+
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/concurrency_groups/list_jobs"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /api/w/test-workspace/concurrency_groups/list_jobs",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/favorites.rs

@@ -0,0 +1,72 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_favorites_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+    // Setup: create a script to favorite
+    let resp = authed(client().post(format!("{ws}/scripts/create")))
+        .json(&json!({
+            "path": "u/test-user/test_fav_script",
+            "summary": "test",
+            "description": "",
+            "content": "export function main() { return 1; }",
+            "language": "deno",
+            "schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /scripts/create (setup)");
+
+    let fav_body = json!({
+        "favorite_kind": "script",
+        "path": "u/test-user/test_fav_script"
+    });
+
+    // POST /favorites/star → 200
+    let resp = authed(client().post(format!("{ws}/favorites/star")))
+        .json(&fav_body)
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /favorites/star");
+
+    // POST /favorites/unstar → 200
+    let resp = authed(client().post(format!("{ws}/favorites/unstar")))
+        .json(&fav_body)
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /favorites/unstar");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/folder_history.rs

@@ -0,0 +1,51 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_folder_history_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    // Create a folder first
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/w/test-workspace/folders/create"
+            ))
+            .json(&json!({"name": "test_hist_folder", "owners": ["u/test-user"]})),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /folders/create");
+
+    // GET /folders_history/get/{folder} → 200 (empty array)
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/w/test-workspace/folders_history/get/test_hist_folder"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /folders_history/get/test_hist_folder");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/granular_acls.rs

@@ -0,0 +1,54 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_granular_acls_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/acls");
+
+    // GET /acls/get/group_/all → 200
+    let resp = authed(client().get(format!("{base}/get/group_/all")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /acls/get/group_/all");
+
+    // POST /acls/add/group_/all → 200
+    let resp = authed(client().post(format!("{base}/add/group_/all")))
+        .json(&json!({"owner": "u/test-user-2", "write": true}))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /acls/add/group_/all");
+
+    // POST /acls/remove/group_/all → 200
+    let resp = authed(client().post(format!("{base}/remove/group_/all")))
+        .json(&json!({"owner": "u/test-user-2"}))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /acls/remove/group_/all");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/group_history.rs

@@ -0,0 +1,32 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_group_history_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/groups_history");
+
+    let resp = authed(client().get(format!("{base}/get/all")))
+        .send()
+        .await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get/all");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/health.rs

@@ -0,0 +1,41 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_health_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/health");
+
+    // GET /health/status → 200 (no auth required)
+    let resp = client().get(format!("{base}/status")).send().await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /health/status");
+
+    // GET /health/detailed → 200 (authed)
+    let resp = authed(client().get(format!("{base}/detailed")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /health/detailed");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/inputs.rs

@@ -0,0 +1,76 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_inputs_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/inputs");
+
+    // GET /history with fake runnable → 200 empty array
+    let resp = authed(client().get(format!(
+        "{base}/history?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/history");
+
+    // GET /list with fake runnable → 200 empty array
+    let resp = authed(client().get(format!(
+        "{base}/list?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/list");
+
+    // POST /create → 200, returns UUID
+    let resp = authed(client().post(format!(
+        "{base}/create?runnable_id=u/test-user/test&runnable_type=ScriptPath"
+    )))
+    .json(&json!({"name": "test_input", "args": {}}))
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /inputs/create");
+    let input_id: String = serde_json::from_str(&body)?;
+
+    // GET /{id}/args → 200
+    let resp = authed(client().get(format!("{base}/{input_id}/args")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /inputs/{id}/args");
+
+    // POST /delete/{id} → 200
+    let resp = authed(client().post(format!("{base}/delete/{input_id}")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /inputs/delete/{id}");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/job_metrics.rs

@@ -0,0 +1,59 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+const FAKE_UUID: &str = "00000000-0000-0000-0000-000000000000";
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_job_metrics_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/job_metrics");
+
+    let resp = authed(client().post(format!("{base}/get/{FAKE_UUID}")))
+        .json(&json!({}))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /get/{id}",
+    );
+
+    let resp = authed(client().post(format!("{base}/set_progress/{FAKE_UUID}")))
+        .json(&json!({"percent": 50}))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /set_progress/{id}",
+    );
+
+    let resp = authed(client().get(format!("{base}/get_progress/{FAKE_UUID}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_progress/{id}",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/jobs_authed.rs

@@ -0,0 +1,309 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use uuid::Uuid;
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+fn assert_route_reachable(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        status != 404 || !body.is_empty(),
+        "Router-level 404 for {endpoint}",
+    );
+}
+
+async fn insert_completed_job(db: &Pool<Postgres>) -> Uuid {
+    let id = Uuid::new_v4();
+    sqlx::query(
+        "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args)
+         VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)",
+    )
+    .bind(id)
+    .execute(db)
+    .await
+    .unwrap();
+    sqlx::query(
+        "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status)
+         VALUES ($1, 'test-workspace', 100, '42'::jsonb, 'success')",
+    )
+    .bind(id)
+    .execute(db)
+    .await
+    .unwrap();
+    id
+}
+
+#[allow(dead_code)]
+async fn create_script(port: u16) -> String {
+    let base = format!("http://localhost:{port}/api/w/test-workspace/scripts");
+    let resp = authed(client().post(format!("{base}/create")))
+        .json(&json!({
+            "path": "u/test-user/test_job_script",
+            "summary": "test",
+            "description": "",
+            "content": "export function main() { return 42; }",
+            "language": "deno",
+            "schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }))
+        .send()
+        .await
+        .unwrap();
+    assert!(
+        resp.status().is_success(),
+        "create script: {}",
+        resp.status()
+    );
+    "u/test-user/test_job_script".to_string()
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_authed_list_and_count(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs");
+
+    // --- List/count endpoints (2xx with empty results) ---
+
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/list",
+    );
+
+    let resp = authed(client().get(format!("{base}/queue/list")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/queue/list",
+    );
+
+    let resp = authed(client().get(format!("{base}/queue/count")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/queue/count",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/list")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/list",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/count")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/count",
+    );
+
+    // --- Global endpoints ---
+
+    let resp = client()
+        .get(format!("http://localhost:{port}/api/jobs/db_clock"))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/db_clock",
+    );
+
+    let resp = authed(client().get(format!(
+        "http://localhost:{port}/api/jobs/completed/count_by_tag"
+    )))
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/count_by_tag",
+    );
+
+    Ok(())
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_authed_completed_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs");
+
+    let job_id = insert_completed_job(&db).await;
+
+    let resp = authed(client().get(format!("{base}/completed/get/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/get",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_result/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/get_result",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_result_maybe/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/get_result_maybe",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_timing/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/completed/get_timing",
+    );
+
+    Ok(())
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_authed_run_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs");
+
+    // Run preview — no pre-existing script needed
+    let resp = authed(client().post(format!("{base}/run/preview")))
+        .json(&json!({
+            "content": "export function main() { return 1; }",
+            "language": "deno",
+            "args": {}
+        }))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /jobs/run/preview",
+    );
+
+    // Run preview flow
+    let resp = authed(client().post(format!("{base}/run/preview_flow")))
+        .json(&json!({
+            "value": {"modules": []},
+            "args": {}
+        }))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /jobs/run/preview_flow",
+    );
+
+    Ok(())
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_authed_reachability(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs");
+    let fake = Uuid::nil();
+
+    // These need complex runtime but should hit the handler (not 404)
+
+    let resp = authed(client().post(format!("{base}/flow/resume/{fake}")))
+        .json(&json!({}))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /jobs/flow/resume",
+    );
+
+    let resp = authed(client().get(format!("{base}/job_signature/{fake}/1")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/job_signature",
+    );
+
+    let resp = authed(client().get(format!("{base}/resume_urls/{fake}/1")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/resume_urls",
+    );
+
+    let resp = authed(client().get(format!("{base}/result_by_id/{fake}/step1")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /jobs/result_by_id",
+    );
+
+    let resp = authed(client().post(format!("{base}/restart/f/{fake}")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /jobs/restart/f",
+    );
+
+    let resp = authed(client().post(format!("{base}/run/workflow_as_code/{fake}/main")))
+        .json(&json!({}))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /jobs/run/workflow_as_code",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/jobs_unauthed.rs

@@ -0,0 +1,250 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use uuid::Uuid;
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+/// Insert a minimal completed job directly into the database for testing.
+async fn insert_completed_job(db: &Pool<Postgres>) -> Uuid {
+    let id = Uuid::new_v4();
+    sqlx::query(
+        "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args)
+         VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)",
+    )
+    .bind(id)
+    .execute(db)
+    .await
+    .unwrap();
+
+    sqlx::query(
+        "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, result, status)
+         VALUES ($1, 'test-workspace', 100, '42'::jsonb, 'success')",
+    )
+    .bind(id)
+    .execute(db)
+    .await
+    .unwrap();
+
+    id
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_unauthed_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u");
+
+    let job_id = insert_completed_job(&db).await;
+
+    // --- No-data endpoints ---
+
+    let resp = authed(client().post(format!("{base}/queue/get_started_at_by_ids")))
+        .json(&json!([]))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /queue/get_started_at_by_ids",
+    );
+
+    // --- Completed job endpoints (unauthed service, with auth header) ---
+
+    let resp = authed(client().get(format!("{base}/get/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get");
+
+    let resp = authed(client().get(format!("{base}/get_logs/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get_logs");
+
+    let resp = authed(client().get(format!("{base}/get_completed_logs_tail/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_completed_logs_tail",
+    );
+
+    let resp = authed(client().get(format!("{base}/get_args/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /get_args");
+
+    let resp = authed(client().get(format!("{base}/completed/get/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /completed/get",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_result/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /completed/get_result",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_result_maybe/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /completed/get_result_maybe",
+    );
+
+    let resp = authed(client().get(format!("{base}/completed/get_timing/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /completed/get_timing",
+    );
+
+    let resp = authed(client().get(format!("{base}/getupdate/{job_id}")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /getupdate",
+    );
+
+    Ok(())
+}
+
+const FAKE_UUID: &str = "00000000-0000-0000-0000-000000000000";
+const FAKE_SECRET: &str = "aabb";
+
+/// Reachability tests for endpoints that need complex runtime.
+/// These just verify the route matches (handler runs), not 2xx.
+fn assert_route_reachable(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        status != 404 || !body.is_empty(),
+        "Router-level 404 for {endpoint}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_jobs_unauthed_complex_reachability(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u");
+
+    let resp = authed(client().get(format!("{base}/resume/{FAKE_UUID}/1/{FAKE_SECRET}")))
+        .send()
+        .await?;
+    assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "GET /resume");
+
+    let resp = authed(client().post(format!("{base}/cancel/{FAKE_UUID}/1/{FAKE_SECRET}")))
+        .send()
+        .await?;
+    assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "POST /cancel");
+
+    let resp = authed(client().get(format!("{base}/get_flow/{FAKE_UUID}/1/{FAKE_SECRET}")))
+        .send()
+        .await?;
+    assert_route_reachable(resp.status().as_u16(), &resp.text().await?, "GET /get_flow");
+
+    let resp = authed(client().post(format!("{base}/queue/cancel/{FAKE_UUID}")))
+        .json(&serde_json::json!({"reason": "test"}))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /queue/cancel",
+    );
+
+    let resp = authed(client().post(format!("{base}/queue/force_cancel/{FAKE_UUID}")))
+        .json(&serde_json::json!({"reason": "test"}))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /queue/force_cancel",
+    );
+
+    let resp = authed(client().post(format!("{base}/flow/resume_suspended/{FAKE_UUID}")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /flow/resume_suspended",
+    );
+
+    let resp = authed(client().get(format!("{base}/flow/approval_info/{FAKE_UUID}")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /flow/approval_info",
+    );
+
+    let resp = authed(client().get(format!("{base}/get_root_job_id/{FAKE_UUID}")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_root_job_id",
+    );
+
+    let resp = authed(client().get(format!("{base}/get_flow_debug_info/{FAKE_UUID}")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_flow_debug_info",
+    );
+
+    let resp = authed(client().get(format!("{base}/get_log_file/{FAKE_UUID}/test.txt")))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_log_file",
+    );
+
+    let resp = authed(client().post(format!("{base}/queue/cancel_persistent/u/test-user/fake")))
+        .json(&serde_json::json!({"reason": "test"}))
+        .send()
+        .await?;
+    assert_route_reachable(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /queue/cancel_persistent",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/npm_proxy.rs

@@ -0,0 +1,85 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+/// Start a mock npm registry that returns valid JSON for any GET request.
+async fn start_mock_registry() -> u16 {
+    use axum::{routing::get, Json, Router};
+
+    let app = Router::new().fallback(get(|| async {
+        Json(json!({
+            "name": "test-package",
+            "versions": {"1.0.0": {"name": "test-package", "version": "1.0.0"}},
+            "dist-tags": {"latest": "1.0.0"}
+        }))
+    }));
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let port = listener.local_addr().unwrap().port();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    port
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_npm_proxy_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/npm_proxy");
+
+    // Start mock npm registry
+    let mock_port = start_mock_registry().await;
+    let mock_url = format!("http://127.0.0.1:{mock_port}");
+
+    // Configure the npm registry to point to our mock
+    let resp = authed(
+        client()
+            .post(format!(
+                "http://localhost:{port}/api/settings/global/npm_config_registry"
+            ))
+            .json(&json!({"value": mock_url})),
+    )
+    .send()
+    .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /settings/global/npm_config_registry",
+    );
+
+    // GET /metadata/{package}
+    let resp = authed(client().get(format!("{base}/metadata/lodash")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /npm_proxy/metadata/lodash",
+    );
+
+    // GET /resolve/{package}
+    let resp = authed(client().get(format!("{base}/resolve/lodash")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /npm_proxy/resolve/lodash",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/raw_apps.rs

@@ -0,0 +1,33 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_raw_apps_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/raw_apps");
+
+    // GET /raw_apps/list → 200 (empty array)
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /raw_apps/list");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/service_logs.rs

@@ -0,0 +1,36 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_service_logs_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/service_logs");
+
+    let resp = authed(client().get(format!("{base}/list_files")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /list_files",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/settings.rs

@@ -0,0 +1,116 @@
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_settings_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/settings");
+
+    let resp = authed(client().get(format!("{base}/envs"))).send().await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /envs");
+
+    let resp = authed(client().get(format!("{base}/global/hub_base_url")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /global/hub_base_url",
+    );
+
+    let resp = authed(client().post(format!("{base}/global/test_key")))
+        .json(&json!({"value": "test"}))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /global/test_key",
+    );
+
+    let resp = authed(client().get(format!("{base}/instance_config")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /instance_config",
+    );
+
+    let resp = authed(client().get(format!("{base}/instance_config/yaml")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /instance_config/yaml",
+    );
+
+    let resp = authed(client().get(format!("{base}/latest_key_renewal_attempt")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /latest_key_renewal_attempt",
+    );
+
+    let resp = authed(client().post(format!("{base}/sync_cached_resource_types")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "POST /sync_cached_resource_types",
+    );
+
+    // --- Reachability only (need external services) ---
+
+    let resp = authed(
+        client()
+            .post(format!("{base}/test_smtp"))
+            .json(&json!({"to": "test@test.com", "subject": "test", "content": "test"})),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert!(
+        status != 404 || !body.is_empty(),
+        "Router-level 404 for POST /test_smtp"
+    );
+
+    let resp = authed(
+        client()
+            .post(format!("{base}/test_license_key"))
+            .json(&json!({"license_key": "fake"})),
+    )
+    .send()
+    .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert!(
+        status != 404 || !body.is_empty(),
+        "Router-level 404 for POST /test_license_key"
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/trash.rs

@@ -0,0 +1,41 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_trash_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/trash");
+
+    // GET /trash/list → 200 (admin, empty array)
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "GET /trash/list");
+
+    // POST /trash/empty → 200 (admin)
+    let resp = authed(client().post(format!("{base}/empty")))
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_2xx(status, &body, "POST /trash/empty");
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs

@@ -14,6 +14,7 @@ use serde_json::json;
 use sqlx::{Pool, Postgres};
 use std::time::Duration;
 
+#[allow(unused_imports)]
 use windmill_test_utils::*;
 
 /// Row shape for querying deployment callback jobs from v2_job_queue
@@ -27,6 +28,7 @@ struct DeploymentCallbackJob {
 }
 
 /// Poll for deployment callback jobs in the queue for a given script path
+#[allow(dead_code)]
 async fn get_deployment_callback_jobs(
     db: &Pool<Postgres>,
     script_path: &str,
@@ -63,6 +65,7 @@ async fn get_deployment_callback_jobs(
 }
 
 /// Configure git sync for the test workspace with workspace dependencies enabled
+#[allow(dead_code)]
 async fn setup_git_sync_config(db: &Pool<Postgres>, sync_script_path: &str) -> anyhow::Result<()> {
     let git_sync_config = json!({
         "include_type": ["workspacedependencies"],
@@ -87,6 +90,7 @@ async fn setup_git_sync_config(db: &Pool<Postgres>, sync_script_path: &str) -> a
 }
 
 /// Create a git repository resource for testing
+#[allow(dead_code)]
 async fn create_git_repo_resource(db: &Pool<Postgres>) -> anyhow::Result<()> {
     sqlx::query(
         r#"
@@ -107,6 +111,7 @@ async fn create_git_repo_resource(db: &Pool<Postgres>) -> anyhow::Result<()> {
 }
 
 /// Create a dummy sync script for testing (with version >= 28103 for debouncing support)
+#[allow(dead_code)]
 async fn create_sync_script(db: &Pool<Postgres>, path: &str) -> anyhow::Result<i64> {
     let hash: i64 = rand::random::<i64>().unsigned_abs() as i64;
     sqlx::query(
@@ -126,6 +131,7 @@ async fn create_sync_script(db: &Pool<Postgres>, path: &str) -> anyhow::Result<i
 }
 
 /// Create a folder for the versioned script path
+#[allow(dead_code)]
 async fn create_folder(db: &Pool<Postgres>, name: &str) -> anyhow::Result<()> {
     sqlx::query(
         r#"

backend/windmill-api-integration-tests/tests/workspace_deps.rs

@@ -0,0 +1,39 @@
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+    reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+    builder.header("Authorization", "Bearer SECRET_TOKEN")
+}
+
+fn assert_2xx(status: u16, body: &str, endpoint: &str) {
+    assert!(
+        (200..300).contains(&status),
+        "{endpoint} returned {status}: {body}",
+    );
+}
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_workspace_deps_2xx(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/workspace_dependencies");
+
+    let resp = authed(client().get(format!("{base}/list"))).send().await?;
+    assert_2xx(resp.status().as_u16(), &resp.text().await?, "GET /list");
+
+    let resp = authed(client().get(format!("{base}/get_latest/python3")))
+        .send()
+        .await?;
+    assert_2xx(
+        resp.status().as_u16(),
+        &resp.text().await?,
+        "GET /get_latest/python3",
+    );
+
+    Ok(())
+}

backend/windmill-api-integration-tests/tests/workspaces.rs

@@ -599,59 +599,60 @@ async fn test_workspace_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
         .unwrap();
     assert_eq!(resp.status(), 200, "tarball: {}", resp.status());
 
-    // ===== Fork operations (on the newly created workspace) =====
+    // ===== Fork operations (EE-only: CE limits workspace count to 2) =====
+    #[cfg(feature = "enterprise")]
+    {
+        let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces");
+        let resp = authed(client().post(format!("{new_ws_base}/create_fork")))
+            .json(&json!({
+                "id": "wm-fork-test-ws",
+                "name": "Forked Test Workspace"
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(resp.status(), 200, "create_fork: {}", resp.text().await?);
 
-    // --- create_fork (workspace-scoped, from new-test-ws) ---
-    let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces");
-    let resp = authed(client().post(format!("{new_ws_base}/create_fork")))
-        .json(&json!({
-            "id": "wm-fork-test-ws",
-            "name": "Forked Test Workspace"
-        }))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(resp.status(), 200, "create_fork: {}", resp.text().await?);
+        // verify fork exists
+        let resp = authed(client().post(format!("{global_base}/exists")))
+            .json(&json!({"id": "wm-fork-test-ws"}))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(resp.json::<bool>().await?, true);
 
-    // verify fork exists
-    let resp = authed(client().post(format!("{global_base}/exists")))
-        .json(&json!({"id": "wm-fork-test-ws"}))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(resp.json::<bool>().await?, true);
+        // --- change_workspace_id ---
+        let fork_ws_base = format!("http://localhost:{port}/api/w/wm-fork-test-ws/workspaces");
+        let resp = authed(client().post(format!("{fork_ws_base}/change_workspace_id")))
+            .json(&json!({
+                "new_id": "wm-fork-renamed",
+                "new_name": "Renamed Fork"
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(
+            resp.status(),
+            200,
+            "change_workspace_id: {}",
+            resp.text().await?
+        );
 
-    // --- change_workspace_id ---
-    let fork_ws_base = format!("http://localhost:{port}/api/w/wm-fork-test-ws/workspaces");
-    let resp = authed(client().post(format!("{fork_ws_base}/change_workspace_id")))
-        .json(&json!({
-            "new_id": "wm-fork-renamed",
-            "new_name": "Renamed Fork"
-        }))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(
-        resp.status(),
-        200,
-        "change_workspace_id: {}",
-        resp.text().await?
-    );
+        // verify renamed workspace exists
+        let resp = authed(client().post(format!("{global_base}/exists")))
+            .json(&json!({"id": "wm-fork-renamed"}))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(resp.json::<bool>().await?, true);
 
-    // verify renamed workspace exists
-    let resp = authed(client().post(format!("{global_base}/exists")))
-        .json(&json!({"id": "wm-fork-renamed"}))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(resp.json::<bool>().await?, true);
-
-    // clean up renamed fork
-    let resp = authed(client().delete(format!("{global_base}/delete/wm-fork-renamed")))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(resp.status(), 200);
+        // clean up renamed fork
+        let resp = authed(client().delete(format!("{global_base}/delete/wm-fork-renamed")))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(resp.status(), 200);
+    }
 
     // --- archive workspace (on the newly created one, not our main test workspace) ---
     let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces");
@@ -803,3 +804,21 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row(
 
     Ok(())
 }
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_get_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces");
+
+    let resp = authed(client().get(format!("{base}/get_imports/u/test-user/nonexistent_script")))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(resp.status(), 200);
+    let imports = resp.json::<Vec<String>>().await?;
+    assert!(imports.is_empty());
+
+    Ok(())
+}

backend/windmill-api-jobs/src/concurrency_groups.rs

@@ -25,8 +25,8 @@ use uuid::Uuid;
 pub fn global_service() -> Router {
     Router::new()
         .route("/list", get(list_concurrency_groups))
-        .route("/prune/*concurrency_key", delete(prune_concurrency_group))
-        .route("/:job_id/key", get(get_concurrency_key))
+        .route("/prune/{*concurrency_key}", delete(prune_concurrency_group))
+        .route("/{job_id}/key", get(get_concurrency_key))
 }
 
 pub fn workspaced_service() -> Router {

backend/windmill-api-jobs/src/job_metrics.rs

@@ -22,13 +22,13 @@ pub fn workspaced_service() -> Router {
         .allow_origin(Any);
 
     Router::new()
-        .route("/get/:id", post(get_job_metrics).layer(cors.clone()))
+        .route("/get/{id}", post(get_job_metrics).layer(cors.clone()))
         .route(
-            "/set_progress/:id",
+            "/set_progress/{id}",
             post(set_job_progress).layer(cors.clone()),
         )
         .route(
-            "/get_progress/:id",
+            "/get_progress/{id}",
             get(get_job_progress).layer(cors.clone()),
         )
 }

backend/windmill-api-jobs/src/types.rs

@@ -511,18 +511,14 @@ pub struct ResumeUrls {
 
 pub struct QueryOrBody<D>(pub Option<D>);
 
-#[axum::async_trait]
-impl<S, D> FromRequest<S, axum::body::Body> for QueryOrBody<D>
+impl<S, D> FromRequest<S> for QueryOrBody<D>
 where
     D: DeserializeOwned,
     S: Send + Sync,
 {
     type Rejection = Response;
 
-    async fn from_request(
-        req: Request<axum::body::Body>,
-        state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
+    async fn from_request(req: Request, state: &S) -> std::result::Result<Self, Self::Rejection> {
         return if req.method() == axum::http::Method::GET {
             let Query(InPayload { payload }) = Query::from_request(req, state)
                 .await

backend/windmill-api-npm-proxy/src/lib.rs

@@ -119,10 +119,10 @@ struct FileEntry {
 pub fn workspaced_service() -> Router {
     Router::new()
         // Use wildcards for package names to support scoped packages like @scope/package
-        .route("/metadata/*package", get(get_package_metadata))
-        .route("/resolve/*package", get(resolve_package_version))
-        .route("/filetree/*package_version", get(get_package_filetree))
-        .route("/file/*package_version_filepath", get(get_package_file))
+        .route("/metadata/{*package}", get(get_package_metadata))
+        .route("/resolve/{*package}", get(resolve_package_version))
+        .route("/filetree/{*package_version}", get(get_package_filetree))
+        .route("/file/{*package_version_filepath}", get(get_package_file))
         .layer(
             CorsLayer::new()
                 .allow_origin(Any)

backend/windmill-api-schedule/src/lib.rs

@@ -56,12 +56,12 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_schedule))
         .route("/list_with_jobs", get(list_schedule_with_jobs))
-        .route("/get/*path", get(get_schedule))
-        .route("/exists/*path", get(exists_schedule))
+        .route("/get/{*path}", get(get_schedule))
+        .route("/exists/{*path}", get(exists_schedule))
         .route("/create", post(create_schedule))
-        .route("/update/*path", post(edit_schedule))
-        .route("/delete/*path", delete(delete_schedule))
-        .route("/setenabled/*path", post(set_enabled))
+        .route("/update/{*path}", post(edit_schedule))
+        .route("/delete/{*path}", delete(delete_schedule))
+        .route("/setenabled/{*path}", post(set_enabled))
         .route("/setdefaulthandler", post(set_default_error_handler))
     // .route("/catchup/*path", post(do_catchup).get(list_catchup))
 }

backend/windmill-api-scripts/src/scripts.rs

@@ -190,18 +190,18 @@ impl ScriptWDraft<ScriptRunnableSettingsHandle> {
 pub fn global_service() -> Router {
     Router::new()
         .route("/hub/top", get(get_top_hub_scripts))
-        .route("/hub/get/*path", get(get_hub_script_by_path))
-        .route("/hub/get_full/*path", get(get_full_hub_script_by_path))
-        .route("/hub/pick/*path", get(pick_hub_script_by_path))
+        .route("/hub/get/{*path}", get(get_hub_script_by_path))
+        .route("/hub/get_full/{*path}", get(get_full_hub_script_by_path))
+        .route("/hub/pick/{*path}", get(pick_hub_script_by_path))
 }
 
 pub fn global_unauthed_service() -> Router {
     Router::new()
         .route(
-            "/tokened_raw/:workspace/:token/*path",
+            "/tokened_raw/{workspace}/{token}/{*path}",
             get(get_tokened_raw_script_by_path),
         )
-        .route("/empty_ts/*path", get(get_empty_ts_script_by_path))
+        .route("/empty_ts/{*path}", get(get_empty_ts_script_by_path))
 }
 
 pub fn workspaced_service() -> Router {
@@ -210,33 +210,33 @@ pub fn workspaced_service() -> Router {
         .route("/list_search", get(list_search_scripts))
         .route("/create", post(create_script))
         .route("/create_snapshot", post(create_snapshot_script))
-        .route("/archive/p/*path", post(archive_script_by_path))
-        .route("/get/draft/*path", get(get_script_by_path_w_draft))
-        .route("/get/p/*path", get(get_script_by_path))
-        .route("/list_tokens/*path", get(list_tokens))
-        .route("/raw/p/*path", get(raw_script_by_path))
-        .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned))
-        .route("/exists/p/*path", get(exists_script_by_path))
-        .route("/archive/h/:hash", post(archive_script_by_hash))
-        .route("/delete/h/:hash", post(delete_script_by_hash))
-        .route("/delete/p/*path", post(delete_script_by_path))
+        .route("/archive/p/{*path}", post(archive_script_by_path))
+        .route("/get/draft/{*path}", get(get_script_by_path_w_draft))
+        .route("/get/p/{*path}", get(get_script_by_path))
+        .route("/list_tokens/{*path}", get(list_tokens))
+        .route("/raw/p/{*path}", get(raw_script_by_path))
+        .route("/raw_unpinned/p/{*path}", get(raw_script_by_path_unpinned))
+        .route("/exists/p/{*path}", get(exists_script_by_path))
+        .route("/archive/h/{hash}", post(archive_script_by_hash))
+        .route("/delete/h/{hash}", post(delete_script_by_hash))
+        .route("/delete/p/{*path}", post(delete_script_by_path))
         .route("/delete_bulk", delete(delete_scripts_bulk))
-        .route("/get/h/:hash", get(get_script_by_hash))
-        .route("/raw/h/:hash", get(raw_script_by_hash))
-        .route("/deployment_status/h/:hash", get(get_deployment_status))
+        .route("/get/h/{hash}", get(get_script_by_hash))
+        .route("/raw/h/{hash}", get(raw_script_by_hash))
+        .route("/deployment_status/h/{hash}", get(get_deployment_status))
         .route("/list_paths", get(list_paths))
         .route(
-            "/toggle_workspace_error_handler/p/*path",
+            "/toggle_workspace_error_handler/p/{*path}",
             post(toggle_workspace_error_handler),
         )
-        .route("/history/p/*path", get(get_script_history))
-        .route("/get_latest_version/*path", get(get_latest_version))
+        .route("/history/p/{*path}", get(get_script_history))
+        .route("/get_latest_version/{*path}", get(get_latest_version))
         .route(
-            "/list_paths_from_workspace_runnable/*path",
+            "/list_paths_from_workspace_runnable/{*path}",
             get(list_paths_from_workspace_runnable),
         )
         .route(
-            "/history_update/h/:hash/p/*path",
+            "/history_update/h/{hash}/p/{*path}",
             post(update_script_history),
         )
         .route("/list_dedicated_with_deps", get(list_dedicated_with_deps))
@@ -1447,7 +1447,7 @@ async fn get_script_history(
     check_scopes(&authed, || format!("scripts:read:{}", path))?;
     let mut tx = user_db.begin(&authed).await?;
     let query_result = sqlx::query!(
-        "SELECT s.hash as hash, dm.deployment_msg as deployment_msg 
+        "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at
         FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash
         WHERE s.workspace_id = $1 AND s.path = $2
         ORDER by s.created_at DESC",
@@ -1463,6 +1463,7 @@ async fn get_script_history(
         .map(|row| ScriptHistory {
             script_hash: ScriptHash(row.hash),
             deployment_msg: row.deployment_msg,
+            created_at: Some(row.created_at),
         })
         .collect();
     return Ok(Json(result));
@@ -1477,7 +1478,7 @@ async fn get_latest_version(
     check_scopes(&authed, || format!("scripts:read:{}", path))?;
     let mut tx = user_db.begin(&authed).await?;
     let row_o = sqlx::query!(
-        "SELECT s.hash as hash, dm.deployment_msg as deployment_msg 
+        "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at
         FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash
         WHERE s.workspace_id = $1 AND s.path = $2
         ORDER by s.created_at DESC LIMIT 1",
@@ -1491,7 +1492,8 @@ async fn get_latest_version(
     if let Some(row) = row_o {
         let result = ScriptHistory {
             script_hash: ScriptHash(row.hash),
-            deployment_msg: row.deployment_msg, //
+            deployment_msg: row.deployment_msg,
+            created_at: Some(row.created_at),
         };
         return Ok(Json(Some(result)));
     } else {

backend/windmill-api-settings/src/lib.rs

@@ -58,7 +58,7 @@ pub fn global_service() -> Router {
     let r = Router::new()
         .route("/envs", get(get_local_settings))
         .route(
-            "/global/:key",
+            "/global/{key}",
             post(set_global_setting).get(get_global_setting),
         )
         .route("/list_global", get(list_global_settings))
@@ -80,7 +80,7 @@ pub fn global_service() -> Router {
         .route("/test_critical_channels", post(test_critical_channels))
         .route("/critical_alerts", get(get_critical_alerts))
         .route(
-            "/critical_alerts/:id/acknowledge",
+            "/critical_alerts/{id}/acknowledge",
             post(acknowledge_critical_alert),
         )
         .route(
@@ -92,7 +92,7 @@ pub fn global_service() -> Router {
             post(refresh_custom_instance_user_pwd),
         )
         .route(
-            "/setup_custom_instance_pg_database/:name",
+            "/setup_custom_instance_pg_database/{name}",
             post(setup_custom_instance_pg_database),
         )
         .route(

backend/windmill-api-users/src/lib.rs

@@ -1 +1,4 @@
 pub mod users;
+#[cfg(feature = "private")]
+pub mod users_ee;
+mod users_oss;

backend/windmill-api-users/src/users.rs

@@ -58,7 +58,7 @@ use windmill_common::{
 use windmill_common::{BASE_URL, HUB_BASE_URL};
 use windmill_git_sync::handle_deployment_metadata;
 
-const COOKIE_PATH: &str = "/";
+pub const COOKIE_PATH: &str = "/";
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -66,32 +66,37 @@ pub fn workspaced_service() -> Router {
         .route("/list_usage", get(list_user_usage))
         .route("/list_usernames", get(list_usernames))
         .route("/exists", post(exists_username))
-        .route("/get/:user", get(get_workspace_user))
-        .route("/update/:user", post(update_workspace_user))
-        .route("/delete/:user", delete(delete_workspace_user))
-        .route("/convert_to_group/:user", post(convert_user_to_group))
-        .route("/is_owner/*path", get(is_owner_of_path))
-        .route("/whois/:username", get(whois))
+        .route("/get/{user}", get(get_workspace_user))
+        .route("/update/{user}", post(update_workspace_user))
+        .route("/delete/{user}", delete(delete_workspace_user))
+        .route("/convert_to_group/{user}", post(convert_user_to_group))
+        .route("/is_owner/{*path}", get(is_owner_of_path))
+        .route("/whois/{username}", get(whois))
         .route("/whoami", get(whoami))
         .route("/leave", post(leave_workspace))
-        .route("/username_to_email/:username", get(username_to_email))
+        .route("/username_to_email/{username}", get(username_to_email))
+        .route(
+            "/impersonate_service_account",
+            post(impersonate_service_account),
+        )
+        .route("/exit_impersonation", post(exit_impersonation))
 }
 
 pub fn global_service() -> Router {
     Router::new()
-        .route("/exists/:email", get(exists_email))
+        .route("/exists/{email}", get(exists_email))
         .route("/email", get(get_email))
         .route("/whoami", get(global_whoami))
         .route("/list_invites", get(list_invites))
         .route("/decline_invite", post(decline_invite))
         .route("/accept_invite", post(accept_invite))
         .route("/list_as_super_admin", get(list_users_as_super_admin))
-        .route("/set_login_type/:user", post(set_login_type))
-        .route("/update/:user", post(update_user))
-        .route("/delete/:user", delete(delete_user))
-        .route("/username_info/:user", get(get_instance_username_info))
+        .route("/set_login_type/{user}", post(set_login_type))
+        .route("/update/{user}", post(update_user))
+        .route("/delete/{user}", delete(delete_user))
+        .route("/username_info/{user}", get(get_instance_username_info))
         .route("/tokens/create", post(create_token))
-        .route("/tokens/delete/:token_prefix", delete(delete_token))
+        .route("/tokens/delete/{token_prefix}", delete(delete_token))
         .route("/tokens/list", get(list_tokens))
         .route("/tokens/impersonate", post(impersonate))
         .route("/usage", get(get_usage))
@@ -135,6 +140,7 @@ pub struct User {
     pub role: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub added_via: Option<serde_json::Value>,
+    pub is_service_account: bool,
 }
 
 #[derive(Serialize)]
@@ -176,6 +182,7 @@ pub struct UserInfo {
     pub folders: Vec<String>,
     pub folders_owners: Vec<String>,
     pub name: Option<String>,
+    pub is_service_account: bool,
 }
 
 #[derive(FromRow, Serialize)]
@@ -620,8 +627,9 @@ async fn is_valid_logout_redirect(rd: &str) -> bool {
 async fn whoami(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-    ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed,
+    authed: ApiAuthed,
 ) -> JsonResult<UserInfo> {
+    let ApiAuthed { username, email, is_admin, groups, folders, .. } = authed;
     let user = get_user(&w_id, &username, &db).await?;
     if let Some(user) = user {
         Ok(Json(user))
@@ -648,6 +656,7 @@ async fn whoami(
                 .into_iter()
                 .filter_map(|x| if x.2 { Some(x.0) } else { None })
                 .collect(),
+            is_service_account: false,
         }))
     }
 }
@@ -663,11 +672,11 @@ async fn global_whoami(
          email = $1",
         email
     )
-    .fetch_one(&db)
+    .fetch_optional(&db)
     .await
-    .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}")));
+    .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}")))?;
 
-    if let Ok(user) = user {
+    if let Some(user) = user {
         Ok(Json(user))
     } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) {
         Ok(Json(GlobalUserInfo {
@@ -685,7 +694,21 @@ async fn global_whoami(
             disabled: false,
         }))
     } else {
-        Err(user.unwrap_err())
+        // Service accounts don't have a password row
+        Ok(Json(GlobalUserInfo {
+            email: email.clone(),
+            login_type: Some("service_account".to_string()),
+            super_admin: false,
+            devops: false,
+            verified: true,
+            name: None,
+            company: None,
+            username: None,
+            operator_only: Some(true),
+            first_time_user: false,
+            role_source: "service_account".to_string(),
+            disabled: false,
+        }))
     }
 }
 
@@ -736,12 +759,13 @@ pub struct User2 {
     pub name: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub added_via: Option<serde_json::Value>,
+    pub is_service_account: bool,
 }
 
 async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
     let user = sqlx::query_as!(
         User2,
-        "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2
+        "SELECT usr.*, COALESCE(password.super_admin, false) as \"super_admin!\", password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2
         ",
         username,
         w_id
@@ -782,6 +806,7 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
             .into_iter()
             .filter_map(|x| if x.2 { Some(x.0) } else { None })
             .collect(),
+        is_service_account: usr.is_service_account,
     }))
 }
 
@@ -2025,6 +2050,44 @@ async fn impersonate(
     Ok((StatusCode::CREATED, token))
 }
 
+#[derive(Deserialize)]
+pub struct ImpersonateServiceAccountRequest {
+    pub username: String,
+}
+
+async fn impersonate_service_account(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    cookies: Cookies,
+    Tokened { token: current_token }: Tokened,
+    Path(w_id): Path<String>,
+    Json(req): Json<ImpersonateServiceAccountRequest>,
+) -> Result<(StatusCode, String)> {
+    crate::users_oss::impersonate_service_account(db, authed, cookies, current_token, w_id, req)
+        .await
+}
+
+#[derive(Deserialize)]
+struct ExitImpersonationRequest {
+    token: String,
+}
+
+async fn exit_impersonation(
+    cookies: Cookies,
+    Json(req): Json<ExitImpersonationRequest>,
+) -> Result<String> {
+    let mut cookie = tower_cookies::Cookie::new(COOKIE_NAME, req.token);
+    cookie.set_secure(IS_SECURE.read().await.clone());
+    cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax));
+    cookie.set_http_only(true);
+    cookie.set_path(COOKIE_PATH);
+    if COOKIE_DOMAIN.is_some() {
+        cookie.set_domain(COOKIE_DOMAIN.clone().unwrap());
+    }
+    cookies.add(cookie);
+    Ok("exited impersonation".to_string())
+}
+
 #[derive(Deserialize)]
 struct ListTokenQuery {
     exclude_ephemeral: Option<bool>,

backend/windmill-api-users/src/users_oss.rs

@@ -0,0 +1,28 @@
+#[cfg(feature = "private")]
+#[allow(unused)]
+pub use crate::users_ee::*;
+
+#[cfg(not(feature = "private"))]
+use crate::users::ImpersonateServiceAccountRequest;
+#[cfg(not(feature = "private"))]
+use http::StatusCode;
+#[cfg(not(feature = "private"))]
+use tower_cookies::Cookies;
+#[cfg(not(feature = "private"))]
+use windmill_api_auth::ApiAuthed;
+#[cfg(not(feature = "private"))]
+use windmill_common::DB;
+
+#[cfg(not(feature = "private"))]
+pub async fn impersonate_service_account(
+    _db: DB,
+    _authed: ApiAuthed,
+    _cookies: Cookies,
+    _current_token: String,
+    _w_id: String,
+    _req: ImpersonateServiceAccountRequest,
+) -> windmill_common::error::Result<(StatusCode, String)> {
+    Err(windmill_common::error::Error::BadRequest(
+        "Service accounts require Windmill Enterprise Edition".to_string(),
+    ))
+}

backend/windmill-api-workspaces/src/workspaces.rs

@@ -75,11 +75,12 @@ pub fn workspaced_service() -> Router {
         .route("/archive", post(archive_workspace))
         .route("/invite_user", post(invite_user))
         .route("/add_user", post(add_user))
+        .route("/create_service_account", post(create_service_account))
         .route("/delete_invite", post(delete_invite))
         .route("/rebuild_dependency_map", post(rebuild_dependency_map))
         .route("/get_dependency_map", get(get_dependency_map))
-        .route("/get_dependents/*imported_path", get(get_dependents))
-        .route("/get_imports/*importer_path", get(get_imports))
+        .route("/get_dependents/{*imported_path}", get(get_dependents))
+        .route("/get_imports/{*importer_path}", get(get_imports))
         .route("/get_dependents_amounts", post(get_dependents_amounts))
         .route("/get_settings", get(get_settings))
         .route(
@@ -152,14 +153,14 @@ pub fn workspaced_service() -> Router {
             post(create_workspace_fork_branch),
         )
         .route(
-            "/reset_diff_tally/:fork_workspace_id",
+            "/reset_diff_tally/{fork_workspace_id}",
             post(reset_workspace_diffs),
         )
-        .route("/compare/:target_workspace_id", get(compare_workspaces))
+        .route("/compare/{target_workspace_id}", get(compare_workspaces))
         .route("/protection_rules", get(list_protection_rules))
         .route("/protection_rules", post(create_protection_rule))
         .route(
-            "/protection_rules/:rule_name",
+            "/protection_rules/{rule_name}",
             post(update_protection_rule).delete(delete_protection_rule),
         )
         .route("/log_chat", post(log_ai_chat))
@@ -176,9 +177,9 @@ pub fn global_service() -> Router {
         .route("/exists", post(exists_workspace))
         .route("/exists_username", post(exists_username))
         .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain))
-        .route("/unarchive/:workspace", post(unarchive_workspace))
+        .route("/unarchive/{workspace}", post(unarchive_workspace))
         .route(
-            "/delete/:workspace",
+            "/delete/{workspace}",
             delete(crate::workspaces_extra::delete_workspace),
         )
         .route(
@@ -652,25 +653,23 @@ async fn get_settings(
 }
 
 async fn get_copilot_settings_state(
-    authed: ApiAuthed,
+    _authed: ApiAuthed,
     Path(w_id): Path<String>,
-    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
 ) -> JsonResult<CopilotSettingsState> {
-    let mut tx = user_db.begin(&authed).await?;
     let workspace_ai_config = sqlx::query_scalar!(
         "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1",
         &w_id
     )
-    .fetch_optional(&mut *tx)
+    .fetch_optional(&db)
     .await
     .map_err(|e| Error::internal_err(format!("getting workspace ai settings: {e:#}")))?;
     let workspace_ai_config = not_found_if_none(workspace_ai_config, "workspace settings", &w_id)?;
     let instance_ai_config: Option<serde_json::Value> =
         sqlx::query_scalar("SELECT value FROM global_settings WHERE name = 'ai_config'")
-            .fetch_optional(&mut *tx)
+            .fetch_optional(&db)
             .await
             .map_err(|e| Error::internal_err(format!("getting instance ai settings: {e:#}")))?;
-    tx.commit().await?;
 
     Ok(Json(build_copilot_settings_state(
         has_ai_providers(workspace_ai_config.as_ref()),
@@ -4234,6 +4233,20 @@ If you do not have an account on {}, login with SSO or ask an admin to create an
     ))
 }
 
+#[derive(Deserialize)]
+pub struct NewServiceAccount {
+    pub username: String,
+}
+
+async fn create_service_account(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(nu): Json<NewServiceAccount>,
+) -> Result<(StatusCode, String)> {
+    crate::workspaces_oss::create_service_account(authed, db, w_id, nu).await
+}
+
 async fn delete_invite(
     ApiAuthed { username, is_admin, .. }: ApiAuthed,
     Extension(db): Extension<DB>,

backend/windmill-api-workspaces/src/workspaces_oss.rs

@@ -3,7 +3,9 @@
 pub use crate::workspaces_ee::*;
 
 #[cfg(not(feature = "private"))]
-use crate::workspaces::EditAutoInvite;
+use crate::workspaces::{EditAutoInvite, NewServiceAccount};
+#[cfg(not(feature = "private"))]
+use http::StatusCode;
 #[cfg(not(feature = "private"))]
 use windmill_api_auth::ApiAuthed;
 #[cfg(not(feature = "private"))]
@@ -20,3 +22,15 @@ pub async fn edit_auto_invite(
         "Not implemented on OSS".to_string(),
     ))
 }
+
+#[cfg(not(feature = "private"))]
+pub async fn create_service_account(
+    _authed: ApiAuthed,
+    _db: DB,
+    _w_id: String,
+    _nu: NewServiceAccount,
+) -> windmill_common::error::Result<(StatusCode, String)> {
+    Err(windmill_common::error::Error::BadRequest(
+        "Service accounts require Windmill Enterprise Edition".to_string(),
+    ))
+}

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.666.0
+  version: 1.668.2
   title: Windmill API
 
   contact:
@@ -2125,6 +2125,87 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/workspaces/create_service_account:
+    post:
+      summary: create a service account
+      operationId: createServiceAccount
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                username:
+                  type: string
+              required:
+                - username
+      responses:
+        "201":
+          description: service account created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/users/impersonate_service_account:
+    post:
+      summary: impersonate a service account
+      operationId: impersonateServiceAccount
+      tags:
+        - user
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                username:
+                  type: string
+              required:
+                - username
+      responses:
+        "201":
+          description: impersonation token
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/users/exit_impersonation:
+    post:
+      summary: exit service account impersonation
+      operationId: exitImpersonation
+      tags:
+        - user
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                token:
+                  type: string
+              required:
+                - token
+      responses:
+        "200":
+          description: exited impersonation
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/workspaces/delete_invite:
     post:
       summary: delete user invite
@@ -20123,6 +20204,8 @@ components:
           nullable: true
           allOf:
             - $ref: "#/components/schemas/UserSource"
+        is_service_account:
+          type: boolean
       required:
         - email
         - username

backend/windmill-api/src/ai.rs

@@ -607,11 +607,11 @@ fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> {
 }
 
 pub fn global_service() -> Router {
-    Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy))
+    Router::new().route("/proxy/{*ai}", post(global_proxy).get(global_proxy))
 }
 
 pub fn workspaced_service() -> Router {
-    let router = Router::new().route("/proxy/*ai", post(proxy).get(proxy));
+    let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy));
 
     #[cfg(feature = "bedrock")]
     let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials));

backend/windmill-api/src/apps.rs

@@ -83,48 +83,54 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
         .route("/list_search", get(list_search_apps))
-        .route("/get/p/*path", get(get_app))
-        .route("/get/lite/*path", get(get_app_lite))
-        .route("/get/draft/*path", get(get_app_w_draft))
-        .route("/secret_of/*path", get(get_secret_id))
+        .route("/get/p/{*path}", get(get_app))
+        .route("/get/lite/{*path}", get(get_app_lite))
+        .route("/get/draft/{*path}", get(get_app_w_draft))
+        .route("/secret_of/{*path}", get(get_secret_id))
         .route(
-            "/secret_of_latest_version/*path",
+            "/secret_of_latest_version/{*path}",
             get(get_latest_version_secret_id),
         )
-        .route("/get/v/*id", get(get_app_by_id))
-        .route("/get_data/v/*id", get(get_raw_app_data))
-        .route("/exists/*path", get(exists_app))
-        .route("/update/*path", post(update_app))
-        .route("/update_raw/*path", post(update_app_raw))
-        .route("/delete/*path", delete(delete_app))
+        .route("/get/v/{*id}", get(get_app_by_id))
+        .route("/get_data/v/{*id}", get(get_raw_app_data))
+        .route("/exists/{*path}", get(exists_app))
+        .route("/update/{*path}", post(update_app))
+        .route("/update_raw/{*path}", post(update_app_raw))
+        .route("/delete/{*path}", delete(delete_app))
         .route("/create", post(create_app))
         .route("/create_raw", post(create_app_raw))
-        .route("/history/p/*path", get(get_app_history))
-        .route("/get_latest_version/*path", get(get_latest_version))
-        .route("/history_update/a/:id/v/:version", post(update_app_history))
+        .route("/history/p/{*path}", get(get_app_history))
+        .route("/get_latest_version/{*path}", get(get_latest_version))
         .route(
-            "/list_paths_from_workspace_runnable/:runnable_kind/*path",
+            "/history_update/a/{id}/v/{version}",
+            post(update_app_history),
+        )
+        .route(
+            "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}",
             get(list_paths_from_workspace_runnable),
         )
-        .route("/custom_path_exists/*custom_path", get(custom_path_exists))
+        .route(
+            "/custom_path_exists/{*custom_path}",
+            get(custom_path_exists),
+        )
         .route("/sign_s3_objects", post(sign_s3_objects))
 }
 
 pub fn unauthed_service() -> Router {
     Router::new()
-        .route("/execute_component/*path", post(execute_component))
-        .route("/upload_s3_file/*path", post(upload_s3_file_from_app))
+        .route("/execute_component/{*path}", post(execute_component))
+        .route("/upload_s3_file/{*path}", post(upload_s3_file_from_app))
         .route("/delete_s3_file", delete(delete_s3_file_from_app))
-        .route("/download_s3_file/*path", get(download_s3_file_from_app))
-        .route("/public_app/:secret", get(get_public_app_by_secret))
-        .route("/public_resource/*path", get(get_public_resource))
-        .route("/get_data/v/*id", get(get_raw_app_data))
+        .route("/download_s3_file/{*path}", get(download_s3_file_from_app))
+        .route("/public_app/{secret}", get(get_public_app_by_secret))
+        .route("/public_resource/{*path}", get(get_public_resource))
+        .route("/get_data/v/{*id}", get(get_raw_app_data))
 }
 pub fn global_service() -> Router {
     Router::new()
         .route("/hub/list", get(list_hub_apps))
-        .route("/hub/get/:id", get(get_hub_app_by_id))
-        .route("/hub/get_raw/:id", get(get_hub_raw_app_by_id))
+        .route("/hub/get/{id}", get(get_hub_app_by_id))
+        .route("/hub/get_raw/{id}", get(get_hub_raw_app_by_id))
 }
 
 #[derive(FromRow, Deserialize, Serialize)]

backend/windmill-api/src/args.rs

@@ -451,8 +451,7 @@ where
     }
 }
 
-#[axum::async_trait]
-impl<S> FromRequest<S, axum::body::Body> for RawWebhookArgs
+impl<S> FromRequest<S> for RawWebhookArgs
 where
     S: Send + Sync,
 {

backend/windmill-api/src/audit.rs

@@ -19,7 +19,7 @@ use crate::db::ApiAuthed;
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_audit))
-        .route("/get/:id", get(get_audit))
+        .route("/get/{id}", get(get_audit))
 }
 
 async fn get_audit(

backend/windmill-api/src/capture.rs

@@ -93,22 +93,22 @@ pub fn workspaced_service() -> Router {
     Router::new()
         .route("/set_config", post(set_config))
         .route(
-            "/ping_config/:trigger_kind/:runnable_kind/*path",
+            "/ping_config/{trigger_kind}/{runnable_kind}/{*path}",
             post(ping_config),
         )
-        .route("/get_configs/:runnable_kind/*path", get(get_configs))
-        .route("/list/:runnable_kind/*path", get(list_captures))
+        .route("/get_configs/{runnable_kind}/{*path}", get(get_configs))
+        .route("/list/{runnable_kind}/{*path}", get(list_captures))
         .route(
-            "/move/:runnable_kind/*path",
+            "/move/{runnable_kind}/{*path}",
             post(move_captures_and_configs),
         )
-        .route("/:id", delete(delete_capture))
-        .route("/:id", get(get_capture))
+        .route("/{id}", delete(delete_capture))
+        .route("/{id}", get(get_capture))
 }
 
 pub fn workspaced_unauthed_service() -> Router {
     let router = Router::new().route(
-        "/webhook/:runnable_kind/*path",
+        "/webhook/{runnable_kind}/{*path}",
         head(|| async {}).post(webhook_payload),
     );
 
@@ -118,12 +118,12 @@ pub fn workspaced_unauthed_service() -> Router {
     ))]
     {
         #[cfg(feature = "http_trigger")]
-        let router = router.route("/http/:runnable_kind/:path/*route_path", {
+        let router = router.route("/http/{runnable_kind}/{path}/{*route_path}", {
             head(|| async {}).fallback(http_payload)
         });
 
         #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))]
-        let router = router.route("/gcp/:runnable_kind/*path", post(gcp_payload));
+        let router = router.route("/gcp/{runnable_kind}/{*path}", post(gcp_payload));
 
         router
     }

backend/windmill-api/src/db_health.rs

@@ -0,0 +1,512 @@
+/*
+ * Author: Windmill Labs
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use axum::{extract::Query, routing::get, Extension, Json, Router};
+use serde::{Deserialize, Serialize};
+
+use windmill_common::error::JsonResult;
+
+use crate::db::{ApiAuthed, DB};
+use crate::health::get_pool_stats;
+use crate::health::PoolStats;
+use crate::utils::require_super_admin;
+
+pub fn global_service() -> Router {
+    Router::new().route("/", get(get_db_health))
+}
+
+// --- Response types ---
+
+#[derive(Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HealthLevel {
+    Green,
+    Yellow,
+    Red,
+}
+
+#[derive(Serialize)]
+pub struct DbHealthResponse {
+    pub database_size: DatabaseSizeInfo,
+    pub job_retention: JobRetentionInfo,
+    pub large_results: LargeResultsInfo,
+    pub connection_pool: ConnectionPoolInfo,
+    pub table_maintenance: Vec<TableMaintenanceInfo>,
+    pub slow_queries: Option<SlowQueriesInfo>,
+    pub datatables: Vec<DatatableInfo>,
+}
+
+#[derive(Serialize)]
+pub struct DatabaseSizeInfo {
+    pub total_size_bytes: i64,
+    pub total_size_pretty: String,
+    pub top_tables: Vec<TableSizeInfo>,
+}
+
+#[derive(Serialize)]
+pub struct TableSizeInfo {
+    pub table_name: String,
+    pub total_size_bytes: i64,
+    pub total_size_pretty: String,
+}
+
+#[derive(Serialize)]
+pub struct JobRetentionInfo {
+    pub oldest_completed_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub total_completed_jobs: i64,
+    pub retention_period_secs: Option<i64>,
+    pub status: HealthLevel,
+    pub message: String,
+}
+
+#[derive(Serialize)]
+pub struct LargeResultsInfo {
+    pub top_large_results: Vec<LargeResultRow>,
+    pub avg_result_size_bytes: Option<i64>,
+}
+
+#[derive(Serialize)]
+pub struct LargeResultRow {
+    pub id: uuid::Uuid,
+    pub workspace_id: String,
+    pub runnable_path: Option<String>,
+    pub result_size_bytes: i64,
+    pub completed_at: chrono::DateTime<chrono::Utc>,
+}
+
+#[derive(Serialize)]
+pub struct ConnectionPoolInfo {
+    pub pool: PoolStats,
+    pub pg_active_connections: i64,
+    pub status: HealthLevel,
+    pub message: String,
+}
+
+#[derive(Serialize)]
+pub struct TableMaintenanceInfo {
+    pub table_name: String,
+    pub live_tuples: i64,
+    pub dead_tuples: i64,
+    pub dead_ratio: f64,
+    pub last_autovacuum: Option<chrono::NaiveDateTime>,
+    pub last_autoanalyze: Option<chrono::NaiveDateTime>,
+    pub status: HealthLevel,
+}
+
+#[derive(Serialize)]
+pub struct SlowQueriesInfo {
+    pub queries: Vec<SlowQueryRow>,
+    pub message: Option<String>,
+}
+
+#[derive(Serialize)]
+pub struct SlowQueryRow {
+    pub query: String,
+    pub calls: i64,
+    pub total_exec_time_ms: f64,
+    pub mean_exec_time_ms: f64,
+}
+
+#[derive(Serialize)]
+pub struct DatatableInfo {
+    pub workspace_id: String,
+    pub name: String,
+    pub table_name: String,
+    pub size_bytes: i64,
+    pub size_pretty: String,
+    pub estimated_rows: f64,
+}
+
+// --- Handler ---
+
+#[derive(Deserialize)]
+struct DbHealthQuery {
+    /// Max number of recent completed jobs to scan for large results (default 10000)
+    scan_limit: Option<i64>,
+}
+
+async fn get_db_health(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Query(query): Query<DbHealthQuery>,
+) -> JsonResult<DbHealthResponse> {
+    require_super_admin(&db, &email).await?;
+
+    let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000);
+
+    let (
+        database_size,
+        job_retention,
+        large_results,
+        connection_pool,
+        table_maintenance,
+        slow_queries,
+        datatables,
+    ) = tokio::try_join!(
+        fetch_database_size(&db),
+        fetch_job_retention(&db),
+        fetch_large_results(&db, scan_limit),
+        fetch_connection_pool(&db),
+        fetch_table_maintenance(&db),
+        fetch_slow_queries(&db),
+        fetch_datatables(&db),
+    )?;
+
+    Ok(Json(DbHealthResponse {
+        database_size,
+        job_retention,
+        large_results,
+        connection_pool,
+        table_maintenance,
+        slow_queries,
+        datatables,
+    }))
+}
+
+// --- Diagnostic queries ---
+
+async fn fetch_database_size(db: &DB) -> windmill_common::error::Result<DatabaseSizeInfo> {
+    let row = sqlx::query!(
+        "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty"
+    )
+    .fetch_one(db)
+    .await?;
+
+    let top_tables = sqlx::query_as!(
+        TableSizeInfo,
+        r#"SELECT
+            schemaname || '.' || relname as "table_name!",
+            pg_total_relation_size(relid) as "total_size_bytes!",
+            pg_size_pretty(pg_total_relation_size(relid)) as "total_size_pretty!"
+        FROM pg_catalog.pg_statio_user_tables
+        ORDER BY pg_total_relation_size(relid) DESC
+        LIMIT 15"#
+    )
+    .fetch_all(db)
+    .await?;
+
+    Ok(DatabaseSizeInfo {
+        total_size_bytes: row.size_bytes.unwrap_or(0),
+        total_size_pretty: row.size_pretty.unwrap_or_default(),
+        top_tables,
+    })
+}
+
+async fn fetch_job_retention(db: &DB) -> windmill_common::error::Result<JobRetentionInfo> {
+    let job_row =
+        sqlx::query!("SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed")
+            .fetch_one(db)
+            .await?;
+
+    let retention_row =
+        sqlx::query!("SELECT value FROM global_settings WHERE name = 'retention_period_secs'")
+            .fetch_optional(db)
+            .await?;
+
+    let retention_period_secs: Option<i64> =
+        retention_row.map(|r| r.value).and_then(|v| v.as_i64());
+
+    let oldest = job_row.oldest;
+    let total = job_row.total.unwrap_or(0);
+
+    let (status, message) = if let (Some(oldest_ts), Some(retention_secs)) =
+        (oldest, retention_period_secs)
+    {
+        let age_secs: i64 = (chrono::Utc::now() - oldest_ts).num_seconds();
+        let ratio = if retention_secs > 0 {
+            age_secs as f64 / retention_secs as f64
+        } else {
+            0.0
+        };
+        if ratio <= 2.0 {
+            (
+                HealthLevel::Green,
+                format!(
+                    "Oldest job is {:.1}x the retention period. Cleanup is keeping up.",
+                    ratio
+                ),
+            )
+        } else if ratio <= 5.0 {
+            (
+                HealthLevel::Yellow,
+                format!(
+                    "Oldest job is {:.1}x the retention period. Cleanup may be falling behind.",
+                    ratio
+                ),
+            )
+        } else {
+            (HealthLevel::Red, format!("Oldest job is {:.1}x the retention period. Consider reducing retention or investigating cleanup.", ratio))
+        }
+    } else if oldest.is_some() && retention_period_secs.is_none() {
+        (
+            HealthLevel::Yellow,
+            "No retention_period_secs configured. Old jobs will accumulate.".to_string(),
+        )
+    } else {
+        (HealthLevel::Green, "No completed jobs found.".to_string())
+    };
+
+    Ok(JobRetentionInfo {
+        oldest_completed_at: oldest,
+        total_completed_jobs: total,
+        retention_period_secs,
+        status,
+        message,
+    })
+}
+
+async fn fetch_large_results(
+    db: &DB,
+    scan_limit: i64,
+) -> windmill_common::error::Result<LargeResultsInfo> {
+    let top_large_results = sqlx::query_as!(
+        LargeResultRow,
+        r#"SELECT
+            c.id as "id!",
+            c.workspace_id as "workspace_id!",
+            j.runnable_path as "runnable_path",
+            pg_column_size(c.result) as "result_size_bytes!",
+            c.completed_at as "completed_at!"
+        FROM (
+            SELECT id, workspace_id, result, completed_at
+            FROM v2_job_completed
+            WHERE completed_at > now() - interval '30 days'
+              AND result IS NOT NULL
+            ORDER BY completed_at DESC
+            LIMIT $1
+        ) c
+        LEFT JOIN v2_job j ON j.id = c.id
+        WHERE pg_column_size(c.result) > 1024
+        ORDER BY pg_column_size(c.result) DESC
+        LIMIT 10"#,
+        scan_limit
+    )
+    .fetch_all(db)
+    .await?;
+
+    let avg_row = sqlx::query!(
+        r#"SELECT AVG(pg_column_size(result))::bigint as "avg_size"
+        FROM (
+            SELECT result FROM v2_job_completed
+            WHERE completed_at > now() - interval '30 days'
+              AND result IS NOT NULL
+            ORDER BY completed_at DESC
+            LIMIT $1
+        ) sub"#,
+        scan_limit
+    )
+    .fetch_one(db)
+    .await?;
+
+    Ok(LargeResultsInfo { top_large_results, avg_result_size_bytes: avg_row.avg_size })
+}
+
+async fn fetch_connection_pool(db: &DB) -> windmill_common::error::Result<ConnectionPoolInfo> {
+    let pool = get_pool_stats(db);
+
+    let active_row =
+        sqlx::query!("SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'")
+            .fetch_one(db)
+            .await?;
+
+    let pg_active = active_row.cnt.unwrap_or(0);
+    let utilization = if pool.max_connections > 0 {
+        pool.size as f64 / pool.max_connections as f64
+    } else {
+        0.0
+    };
+
+    let (status, message) = if utilization < 0.8 {
+        (
+            HealthLevel::Green,
+            format!(
+                "Pool utilization: {:.0}% ({}/{})",
+                utilization * 100.0,
+                pool.size,
+                pool.max_connections
+            ),
+        )
+    } else if utilization < 0.95 {
+        (
+            HealthLevel::Yellow,
+            format!(
+                "Pool utilization is high: {:.0}% ({}/{}). Consider increasing max_connections.",
+                utilization * 100.0,
+                pool.size,
+                pool.max_connections
+            ),
+        )
+    } else {
+        (
+            HealthLevel::Red,
+            format!(
+                "Pool near exhaustion: {:.0}% ({}/{}). Increase max_connections urgently.",
+                utilization * 100.0,
+                pool.size,
+                pool.max_connections
+            ),
+        )
+    };
+
+    Ok(ConnectionPoolInfo { pool, pg_active_connections: pg_active, status, message })
+}
+
+async fn fetch_table_maintenance(
+    db: &DB,
+) -> windmill_common::error::Result<Vec<TableMaintenanceInfo>> {
+    let rows = sqlx::query!(
+        r#"SELECT
+            schemaname || '.' || relname as "table_name!",
+            COALESCE(n_live_tup, 0) as "live_tuples!",
+            COALESCE(n_dead_tup, 0) as "dead_tuples!",
+            last_autovacuum as "last_autovacuum",
+            last_autoanalyze as "last_autoanalyze"
+        FROM pg_stat_user_tables
+        ORDER BY n_dead_tup DESC
+        LIMIT 15"#
+    )
+    .fetch_all(db)
+    .await?;
+
+    Ok(rows
+        .into_iter()
+        .map(|r| {
+            let total = r.live_tuples + r.dead_tuples;
+            let dead_ratio = if total > 0 {
+                r.dead_tuples as f64 / total as f64
+            } else {
+                0.0
+            };
+            let status = if dead_ratio < 0.1 {
+                HealthLevel::Green
+            } else if dead_ratio < 0.3 {
+                HealthLevel::Yellow
+            } else {
+                HealthLevel::Red
+            };
+            TableMaintenanceInfo {
+                table_name: r.table_name,
+                live_tuples: r.live_tuples,
+                dead_tuples: r.dead_tuples,
+                dead_ratio,
+                last_autovacuum: r.last_autovacuum.map(|t| t.naive_utc()),
+                last_autoanalyze: r.last_autoanalyze.map(|t| t.naive_utc()),
+                status,
+            }
+        })
+        .collect())
+}
+
+async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result<Option<SlowQueriesInfo>> {
+    let ext_exists: bool = sqlx::query_scalar!(
+        r#"SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as "exists!""#
+    )
+    .fetch_one(db)
+    .await?;
+
+    if !ext_exists {
+        return Ok(Some(SlowQueriesInfo {
+            queries: vec![],
+            message: Some(
+                "pg_stat_statements extension is not installed. Enable it for slow query insights."
+                    .to_string(),
+            ),
+        }));
+    }
+
+    // Use raw query since pg_stat_statements may not exist at compile time
+    let rows: Vec<SlowQueryRow> = sqlx::query_as::<_, (String, i64, f64, f64)>(
+        r#"SELECT
+            LEFT(query, 200),
+            calls::bigint,
+            total_exec_time::float8,
+            mean_exec_time::float8
+        FROM pg_stat_statements
+        WHERE query NOT LIKE '%pg_stat_statements%'
+        ORDER BY mean_exec_time DESC
+        LIMIT 10"#,
+    )
+    .fetch_all(db)
+    .await?
+    .into_iter()
+    .map(
+        |(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow {
+            query,
+            calls,
+            total_exec_time_ms,
+            mean_exec_time_ms,
+        },
+    )
+    .collect();
+
+    Ok(Some(SlowQueriesInfo { queries: rows, message: None }))
+}
+
+async fn fetch_datatables(db: &DB) -> windmill_common::error::Result<Vec<DatatableInfo>> {
+    // Find instance-type datatables from workspace_settings
+    let rows = sqlx::query!(
+        r#"SELECT
+            ws.workspace_id as "workspace_id!",
+            dt.key as "name!",
+            dt.value->>'table_name' as "table_name"
+        FROM workspace_settings ws,
+             jsonb_each(ws.datatable) dt
+        WHERE dt.value->>'resource_type' = 'instance'
+          AND dt.value->>'table_name' IS NOT NULL"#
+    )
+    .fetch_all(db)
+    .await?;
+
+    let table_names: Vec<String> = rows.iter().filter_map(|r| r.table_name.clone()).collect();
+
+    if table_names.is_empty() {
+        return Ok(vec![]);
+    }
+
+    // Batch lookup: single query for all table sizes
+    let size_rows = sqlx::query!(
+        r#"SELECT
+            c.relname as "table_name!",
+            pg_total_relation_size(c.oid) as "size_bytes!",
+            pg_size_pretty(pg_total_relation_size(c.oid)) as "size_pretty!",
+            COALESCE(c.reltuples, 0) as "estimated_rows!"
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = 'public' AND c.relname = ANY($1)"#,
+        &table_names
+    )
+    .fetch_all(db)
+    .await?;
+
+    let size_map: std::collections::HashMap<String, _> = size_rows
+        .into_iter()
+        .map(|s| (s.table_name.clone(), s))
+        .collect();
+
+    let mut result = Vec::new();
+    for row in rows {
+        let table_name = match &row.table_name {
+            Some(t) => t.clone(),
+            None => continue,
+        };
+        if let Some(s) = size_map.get(&table_name) {
+            result.push(DatatableInfo {
+                workspace_id: row.workspace_id,
+                name: row.name,
+                table_name,
+                size_bytes: s.size_bytes,
+                size_pretty: s.size_pretty.clone(),
+                estimated_rows: s.estimated_rows as f64,
+            });
+        }
+    }
+
+    // Sort by size descending
+    result.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes));
+    Ok(result)
+}

backend/windmill-api/src/drafts.rs

@@ -23,7 +23,7 @@ use windmill_common::{db::UserDB, error::Result, utils::StripPath};
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/create", post(create_draft))
-        .route("/delete/:kind/*path", delete(delete_draft))
+        .route("/delete/{kind}/{*path}", delete(delete_draft))
 }
 
 #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]

backend/windmill-api/src/flows.rs

@@ -21,7 +21,7 @@ use windmill_common::{error::JsonResult, utils::StripPath, DB};
 /// that depends on windmill-api internals.
 pub fn workspaced_service() -> Router {
     windmill_api_flows::flows::workspaced_service()
-        .route("/get_triggers_count/*path", get(get_triggers_count))
+        .route("/get_triggers_count/{*path}", get(get_triggers_count))
 }
 
 async fn get_triggers_count(

backend/windmill-api/src/google.rs

@@ -111,17 +111,16 @@ pub async fn handle_google_ai_chat(
 
     let (contents, system_instruction) = openai_messages_to_gemini(&request.messages);
 
-    let generation_config =
-        if request.temperature.is_some() || request.max_tokens.is_some() {
-            Some(GeminiGenerationConfig {
-                temperature: request.temperature,
-                max_output_tokens: request.max_tokens,
-                response_mime_type: None,
-                response_schema: None,
-            })
-        } else {
-            None
-        };
+    let generation_config = if request.temperature.is_some() || request.max_tokens.is_some() {
+        Some(GeminiGenerationConfig {
+            temperature: request.temperature,
+            max_output_tokens: request.max_tokens,
+            response_mime_type: None,
+            response_schema: None,
+        })
+    } else {
+        None
+    };
 
     let gemini_tools = request.tools.as_ref().map(|tools| {
         let declarations: Vec<GeminiFunctionDeclaration> = tools
@@ -136,10 +135,7 @@ pub async fn handle_google_ai_chat(
                 }
             })
             .collect();
-        vec![GeminiTool {
-            function_declarations: Some(declarations),
-            google_search: None,
-        }]
+        vec![GeminiTool { function_declarations: Some(declarations), google_search: None }]
     });
 
     let gemini_request = GeminiTextRequest {
@@ -184,9 +180,10 @@ async fn handle_streaming(
         .body(request_body);
     let request = set_auth(request, api_key, is_vertex);
 
-    let response = request.send().await.map_err(|e| {
-        Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
-    })?;
+    let response = request
+        .send()
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to send request to Gemini API: {}", e)))?;
 
     if let Err(e) = response.error_for_status_ref() {
         let status = e.status().map(|s| s.to_string()).unwrap_or_default();
@@ -273,9 +270,10 @@ pub async fn handle_google_ai_models(
     let request = HTTP_CLIENT.get(&endpoint);
     let request = set_auth(request, api_key, is_vertex);
 
-    let response = request.send().await.map_err(|e| {
-        Error::internal_err(format!("Failed to fetch Gemini models: {}", e))
-    })?;
+    let response = request
+        .send()
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?;
 
     if let Err(e) = response.error_for_status_ref() {
         let status = e.status().map(|s| s.to_string()).unwrap_or_default();
@@ -327,9 +325,10 @@ async fn handle_non_streaming(
         .body(request_body);
     let request = set_auth(request, api_key, is_vertex);
 
-    let response = request.send().await.map_err(|e| {
-        Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
-    })?;
+    let response = request
+        .send()
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to send request to Gemini API: {}", e)))?;
 
     if let Err(e) = response.error_for_status_ref() {
         let status = e.status().map(|s| s.to_string()).unwrap_or_default();
@@ -337,9 +336,10 @@ async fn handle_non_streaming(
         return Err(Error::AIError(format!("{}: {}", status, body)));
     }
 
-    let body = response.bytes().await.map_err(|e| {
-        Error::internal_err(format!("Failed to read Gemini response body: {}", e))
-    })?;
+    let body = response
+        .bytes()
+        .await
+        .map_err(|e| Error::internal_err(format!("Failed to read Gemini response body: {}", e)))?;
 
     let parsed = parse_gemini_response(&body)?;
     let openai_response = gemini_response_to_openai(&parsed, model);

backend/windmill-api/src/group_history.rs

@@ -22,7 +22,7 @@ use serde::Serialize;
 use sqlx::FromRow;
 
 pub fn workspaced_service() -> Router {
-    Router::new().route("/get/:name", get(get_group_permission_history))
+    Router::new().route("/get/{name}", get(get_group_permission_history))
 }
 
 #[derive(Serialize, FromRow)]

backend/windmill-api/src/health.rs

@@ -231,7 +231,7 @@ async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult {
     DatabaseCheckResult { healthy, latency_ms }
 }
 
-fn get_pool_stats(db: &DB) -> PoolStats {
+pub(crate) fn get_pool_stats(db: &DB) -> PoolStats {
     PoolStats {
         size: db.size(),
         idle: db.num_idle() as u32,

backend/windmill-api/src/jobs.rs

@@ -134,14 +134,14 @@ pub fn workspaced_service() -> Router {
 
     Router::new()
         .route(
-            "/run/f/*script_path",
+            "/run/f/{*script_path}",
             post(run_flow_by_path)
                 .head(|| async { "" })
                 .layer(cors.clone())
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run/fv/:version",
+            "/run/fv/{version}",
             post(run_flow_by_version)
                 .head(|| async { "" })
                 .layer(cors.clone())
@@ -155,25 +155,25 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run/workflow_as_code/:job_id/:entrypoint",
+            "/run/workflow_as_code/{job_id}/{entrypoint}",
             post(run_workflow_as_code)
                 .head(|| async { "" })
                 .layer(cors.clone())
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/restart/f/:job_id",
+            "/restart/f/{job_id}",
             post(restart_flow).head(|| async { "" }).layer(cors.clone()),
         )
         .route(
-            "/run/p/*script_path",
+            "/run/p/{*script_path}",
             post(run_script_by_path)
                 .head(|| async { "" })
                 .layer(cors.clone())
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_wait_result/p/*script_path",
+            "/run_wait_result/p/{*script_path}",
             post(run_wait_result_script_by_path)
                 .get(run_wait_result_job_by_path_get)
                 .head(|| async { "" })
@@ -181,14 +181,14 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_wait_result/h/:hash",
+            "/run_wait_result/h/{hash}",
             post(run_wait_result_script_by_hash)
                 .head(|| async { "" })
                 .layer(cors.clone())
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_wait_result/f/*script_path",
+            "/run_wait_result/f/{*script_path}",
             post(run_wait_result_flow_by_path)
                 .get(run_wait_result_flow_by_path_get)
                 .head(|| async { "" })
@@ -196,7 +196,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_wait_result/fv/:version",
+            "/run_wait_result/fv/{version}",
             post(run_wait_result_flow_by_version)
                 .get(run_wait_result_flow_by_version_get)
                 .head(|| async { "" })
@@ -204,7 +204,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_and_stream/f/*script_path",
+            "/run_and_stream/f/{*script_path}",
             get(stream_flow_by_path)
                 .post(stream_flow_by_path)
                 .head(|| async { "" })
@@ -212,7 +212,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_and_stream/fv/:version",
+            "/run_and_stream/fv/{version}",
             get(stream_flow_by_version)
                 .post(stream_flow_by_version)
                 .head(|| async { "" })
@@ -220,7 +220,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_and_stream/p/*script_path",
+            "/run_and_stream/p/{*script_path}",
             get(stream_script_by_path)
                 .post(stream_script_by_path)
                 .head(|| async { "" })
@@ -228,7 +228,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run_and_stream/h/:hash",
+            "/run_and_stream/h/{hash}",
             get(stream_script_by_hash)
                 .post(stream_script_by_hash)
                 .head(|| async { "" })
@@ -236,7 +236,7 @@ pub fn workspaced_service() -> Router {
                 .layer(ce_headers.clone()),
         )
         .route(
-            "/run/h/:hash",
+            "/run/h/{hash}",
             post(run_job_by_hash)
                 .head(|| async { "" })
                 .layer(cors.clone())
@@ -245,10 +245,10 @@ pub fn workspaced_service() -> Router {
         .route("/run/preview", post(run_preview_script))
         .route("/run_inline/preview", post(run_inline_preview_script))
         .route(
-            "/run_inline/p/*script_path",
+            "/run_inline/p/{*script_path}",
             post(run_inline_script_by_path),
         )
-        .route("/run_inline/h/:hash", post(run_inline_script_by_hash))
+        .route("/run_inline/h/{hash}", post(run_inline_script_by_hash))
         .route(
             "/run_wait_result/preview",
             post(run_wait_result_preview_script),
@@ -257,7 +257,7 @@ pub fn workspaced_service() -> Router {
             "/run/preview_bundle",
             post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()),
         )
-        .route("/add_batch_jobs/:n", post(add_batch_jobs))
+        .route("/add_batch_jobs/{n}", post(add_batch_jobs))
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route(
             "/run_wait_result/preview_flow",
@@ -280,8 +280,8 @@ pub fn workspaced_service() -> Router {
         )
         .route("/queue/count", get(count_queue_jobs))
         .route("/queue/list_filtered_uuids", get(list_filtered_uuids))
-        .route("/queue/position/:timestamp", get(get_queue_position))
-        .route("/queue/scheduled_for/:id", get(get_scheduled_for))
+        .route("/queue/position/{timestamp}", get(get_queue_position))
+        .route("/queue/scheduled_for/{id}", get(get_scheduled_for))
         .route("/queue/cancel_selection", post(cancel_selection))
         .route("/completed/count", get(count_completed_jobs))
         .route("/completed/count_jobs", get(count_completed_jobs_detail))
@@ -299,49 +299,49 @@ pub fn workspaced_service() -> Router {
         )
         .route("/delete", post(crate::jobs_export::delete_jobs))
         .route(
-            "/completed/get/:id",
+            "/completed/get/{id}",
             get(get_completed_job).layer(cors.clone()),
         )
         .route(
-            "/completed/get_result/:id",
+            "/completed/get_result/{id}",
             get(get_completed_job_result).layer(cors.clone()),
         )
         .route(
-            "/completed/get_result_maybe/:id",
+            "/completed/get_result_maybe/{id}",
             get(get_completed_job_result_maybe).layer(cors.clone()),
         )
         .route(
-            "/completed/get_timing/:id",
+            "/completed/get_timing/{id}",
             get(get_completed_job_timing).layer(cors.clone()),
         )
         .route(
-            "/completed/delete/:id",
+            "/completed/delete/{id}",
             post(delete_completed_job).layer(cors.clone()),
         )
         .route(
-            "/flow/resume/:id",
+            "/flow/resume/{id}",
             post(resume_suspended_flow_as_owner).layer(cors.clone()),
         )
         .route(
-            "/job_signature/:job_id/:resume_id",
+            "/job_signature/{job_id}/{resume_id}",
             get(create_job_signature).layer(cors.clone()),
         )
         .route(
-            "/flow/user_states/:job_id/:key",
+            "/flow/user_states/{job_id}/{key}",
             get(get_flow_user_state)
                 .post(set_flow_user_state)
                 .layer(cors.clone()),
         )
         .route(
-            "/resume_urls/:job_id/:resume_id",
+            "/resume_urls/{job_id}/{resume_id}",
             get(get_resume_urls).layer(cors.clone()),
         )
         .route(
-            "/result_by_id/:job_id/:node_id",
+            "/result_by_id/{job_id}/{node_id}",
             get(get_result_by_id).layer(cors.clone()),
         )
         .route(
-            "/flow_env_by_flow_job_id/:flow_job_id/:var_name",
+            "/flow_env_by_flow_job_id/{flow_job_id}/{var_name}",
             get(get_flow_env_by_flow_job_id).layer(cors.clone()),
         )
         .route("/run/dependencies", post(run_dependencies_job))
@@ -350,59 +350,59 @@ pub fn workspaced_service() -> Router {
             "/send_email_with_instance_smtp",
             post(send_email_with_instance_smtp),
         )
-        .route("/get_otel_traces/:id", get(get_otel_traces))
+        .route("/get_otel_traces/{id}", get(get_otel_traces))
 }
 
 pub fn workspace_unauthed_service() -> Router {
     Router::new()
         .route(
-            "/resume/:job_id/:resume_id/:secret",
+            "/resume/{job_id}/{resume_id}/{secret}",
             get(resume_suspended_job),
         )
         .route(
-            "/resume/:job_id/:resume_id/:secret",
+            "/resume/{job_id}/{resume_id}/{secret}",
             post(resume_suspended_job),
         )
         .route(
-            "/cancel/:job_id/:resume_id/:secret",
+            "/cancel/{job_id}/{resume_id}/{secret}",
             get(cancel_suspended_job),
         )
         .route(
-            "/cancel/:job_id/:resume_id/:secret",
+            "/cancel/{job_id}/{resume_id}/{secret}",
             post(cancel_suspended_job),
         )
         .route(
-            "/get_flow/:job_id/:resume_id/:secret",
+            "/get_flow/{job_id}/{resume_id}/{secret}",
             get(get_suspended_job_flow),
         )
-        .route("/get_root_job_id/:id", get(get_root_job))
-        .route("/get/:id", get(get_job))
-        .route("/get_logs/:id", get(get_job_logs))
+        .route("/get_root_job_id/{id}", get(get_root_job))
+        .route("/get/{id}", get(get_job))
+        .route("/get_logs/{id}", get(get_job_logs))
         .route(
-            "/get_completed_logs_tail/:id",
+            "/get_completed_logs_tail/{id}",
             get(get_completed_job_logs_tail),
         )
-        .route("/get_args/:id", get(get_args))
+        .route("/get_args/{id}", get(get_args))
         .route("/queue/get_started_at_by_ids", post(get_started_at_by_ids))
-        .route("/get_flow_debug_info/:id", get(get_flow_job_debug_info))
-        .route("/completed/get/:id", get(get_completed_job))
-        .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route("/get_flow_debug_info/{id}", get(get_flow_job_debug_info))
+        .route("/completed/get/{id}", get(get_completed_job))
+        .route("/completed/get_result/{id}", get(get_completed_job_result))
         .route(
-            "/completed/get_result_maybe/:id",
+            "/completed/get_result_maybe/{id}",
             get(get_completed_job_result_maybe),
         )
-        .route("/completed/get_timing/:id", get(get_completed_job_timing))
-        .route("/getupdate/:id", get(get_job_update))
-        .route("/getupdate_sse/:id", get(get_job_update_sse))
-        .route("/get_log_file/*file_path", get(get_log_file))
-        .route("/queue/cancel/:id", post(cancel_job_api))
+        .route("/completed/get_timing/{id}", get(get_completed_job_timing))
+        .route("/getupdate/{id}", get(get_job_update))
+        .route("/getupdate_sse/{id}", get(get_job_update_sse))
+        .route("/get_log_file/{*file_path}", get(get_log_file))
+        .route("/queue/cancel/{id}", post(cancel_job_api))
         .route(
-            "/queue/cancel_persistent/*script_path",
+            "/queue/cancel_persistent/{*script_path}",
             post(cancel_persistent_script_api),
         )
-        .route("/queue/force_cancel/:id", post(force_cancel))
-        .route("/flow/resume_suspended/:job_id", post(resume_suspended))
-        .route("/flow/approval_info/:job_id", get(get_approval_info))
+        .route("/queue/force_cancel/{id}", post(force_cancel))
+        .route("/flow/resume_suspended/{job_id}", post(resume_suspended))
+        .route("/flow/approval_info/{job_id}", get(get_approval_info))
 }
 
 pub fn global_root_service() -> Router {

backend/windmill-api/src/lib.rs

@@ -37,6 +37,7 @@ use axum::body::Body;
 use axum::extract::DefaultBodyLimit;
 use axum::http::HeaderValue;
 use axum::response::Response;
+use axum::serve::ListenerExt;
 use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router};
 use db::DB;
 use tokio::task::JoinHandle;
@@ -76,6 +77,7 @@ mod bedrock;
 mod capture;
 mod concurrency_groups;
 mod db;
+mod db_health;
 mod google;
 
 mod drafts;
@@ -473,14 +475,17 @@ pub async fn run_server(
             let (mcp_router, mcp_cancellation_token) =
                 setup_mcp_server(db.clone(), user_db, _base_internal_url.clone()).await?;
             // Workspace-scoped MCP router
+            // Use `layer` instead of `route_layer` because the MCP router only has
+            // a fallback_service (no explicit routes), and axum 0.8 panics on
+            // route_layer with no routes.
             let workspaced_mcp_router = mcp_router
                 .clone()
-                .route_layer(from_extractor::<ApiAuthed>())
+                .layer(from_extractor::<ApiAuthed>())
                 .layer(axum::middleware::from_fn(add_www_authenticate_header))
                 .layer(axum::middleware::from_fn(extract_and_store_workspace_id));
             // Gateway MCP router — resolves workspace from token
             let gateway_mcp_router = mcp_router
-                .route_layer(from_extractor::<ApiAuthed>())
+                .layer(from_extractor::<ApiAuthed>())
                 .layer(axum::middleware::from_fn(
                     add_www_authenticate_header_gateway,
                 ))
@@ -527,7 +532,7 @@ pub async fn run_server(
             "/api",
             Router::new()
                 .nest(
-                    "/w/:workspace_id",
+                    "/w/{workspace_id}",
                     Router::new()
                         // Reordered alphabetically
                         .nest("/acls", granular_acls::workspaced_service())
@@ -640,7 +645,8 @@ pub async fn run_server(
                 .nest("/ai", ai::global_service())
                 .nest("/inkeep", inkeep_oss::global_service())
                 .nest("/indexer", indexer_oss::management_service())
-                .nest("/mcp/w/:workspace_id/list_tools", mcp_list_tools_service)
+                .nest("/mcp/w/{workspace_id}/list_tools", mcp_list_tools_service)
+                .nest("/db_health", db_health::global_service())
                 .nest("/health/detailed", health::detailed_service())
                 .nest(
                     "/saml",
@@ -659,7 +665,7 @@ pub async fn run_server(
                 .route_layer(from_extractor::<users::Tokened>())
                 // Workspace-scoped OAuth endpoints that don't require authentication
                 // (authorize and token are called by MCP client before user is authenticated)
-                .nest("/w/:workspace_id/mcp/oauth/server", {
+                .nest("/w/{workspace_id}/mcp/oauth/server", {
                     #[cfg(feature = "mcp")]
                     {
                         mcp::oauth_server::workspaced_unauthed_service()
@@ -680,7 +686,7 @@ pub async fn run_server(
                 })
                 .nest("/jobs", jobs::global_root_service())
                 .nest(
-                    "/srch/w/:workspace_id/index",
+                    "/srch/w/{workspace_id}/index",
                     indexer_oss::workspaced_service(),
                 )
                 .nest("/srch/index", indexer_oss::global_service())
@@ -710,19 +716,19 @@ pub async fn run_server(
                     }
                 })
                 .nest(
-                    "/w/:workspace_id/apps_u",
+                    "/w/{workspace_id}/apps_u",
                     apps::unauthed_service()
                         .layer(from_extractor::<OptAuthed>())
                         .layer(cors.clone()),
                 )
                 .layer(from_extractor::<OptAuthed>())
-                // Deprecated, here for backwards compatibility: user should use /mcp/w/:workspace_id/mcp instead
+                // Deprecated, here for backwards compatibility: user should use /mcp/w/{workspace_id}/mcp instead
                 .nest(
-                    "/mcp/w/:workspace_id/sse",
+                    "/mcp/w/{workspace_id}/sse",
                     mcp_router.clone().layer(cors.clone()),
                 )
                 .nest(
-                    "/mcp/w/:workspace_id/mcp",
+                    "/mcp/w/{workspace_id}/mcp",
                     mcp_router.clone().layer(cors.clone()),
                 )
                 .nest("/mcp/gateway", gateway_mcp_router.layer(cors.clone()))
@@ -745,7 +751,7 @@ pub async fn run_server(
                         Router::new()
                     }
                 })
-                .nest("/w/:workspace_id/agent_workers", {
+                .nest("/w/{workspace_id}/agent_workers", {
                     #[cfg(feature = "agent_worker_server")]
                     {
                         agent_workers_router
@@ -762,7 +768,7 @@ pub async fn run_server(
                     }
                 })
                 .nest(
-                    "/w/:workspace_id/jobs_u",
+                    "/w/{workspace_id}/jobs_u",
                     jobs::workspace_unauthed_service().layer(cors.clone()),
                 )
                 .route("/slack", post(slack_approvals::slack_app_callback_handler))
@@ -778,14 +784,14 @@ pub async fn run_server(
                     }
                 })
                 .route(
-                    "/w/:workspace_id/jobs/slack_approval/:job_id",
+                    "/w/{workspace_id}/jobs/slack_approval/{job_id}",
                     get(slack_approvals::request_slack_approval),
                 )
                 .route(
-                    "/w/:workspace_id/jobs/teams_approval/:job_id",
+                    "/w/{workspace_id}/jobs/teams_approval/{job_id}",
                     get(teams_approvals_oss::request_teams_approval),
                 )
-                .nest("/w/:workspace_id/github_app", {
+                .nest("/w/{workspace_id}/github_app", {
                     #[cfg(feature = "enterprise")]
                     {
                         git_sync_oss::workspaced_service()
@@ -804,14 +810,14 @@ pub async fn run_server(
                     Router::new()
                 })
                 .nest(
-                    "/w/:workspace_id/resources_u",
+                    "/w/{workspace_id}/resources_u",
                     public_service().layer(cors.clone()),
                 )
                 .nest(
-                    "/w/:workspace_id/capture_u",
+                    "/w/{workspace_id}/capture_u",
                     capture::workspaced_unauthed_service().layer(cors.clone()),
                 )
-                .nest("/w/:workspace_id/s3_proxy", {
+                .nest("/w/{workspace_id}/s3_proxy", {
                     s3_proxy_oss::workspaced_unauthed_service()
                 })
                 .nest(
@@ -856,7 +862,7 @@ pub async fn run_server(
                         Router::new()
                     }
                 })
-                .nest("/gcp/w/:workspace_id", {
+                .nest("/gcp/w/{workspace_id}", {
                     #[cfg(all(
                         feature = "enterprise",
                         feature = "gcp_trigger",
@@ -883,10 +889,10 @@ pub async fn run_server(
                 .route("/openapi.json", get(openapi_json)),
         )
         // Clients must use workspace-scoped OAuth metadata at:
-        // /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server
+        // /.well-known/oauth-authorization-server/api/w/{workspace_id}/mcp/oauth/server
         // This is discovered via /.well-known/oauth-protected-resource?workspace_id=...
         .route(
-            "/.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server",
+            "/.well-known/oauth-authorization-server/api/w/{workspace_id}/mcp/oauth/server",
             {
                 #[cfg(feature = "mcp")]
                 {
@@ -898,9 +904,9 @@ pub async fn run_server(
                 }
             },
         )
-        // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp
+        // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/{workspace_id}/mcp
         .route(
-            "/.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp",
+            "/.well-known/oauth-protected-resource/api/mcp/w/{workspace_id}/mcp",
             {
                 #[cfg(feature = "mcp")]
                 {
@@ -976,7 +982,10 @@ pub async fn run_server(
     if let Some(name) = name.as_ref() {
         tracing::info!("server starting for name={name}");
     }
-    let server = axum::serve(listener, app.into_make_service()).tcp_nodelay(!server_mode);
+    let listener = listener.tap_io(move |tcp_stream| {
+        let _ = tcp_stream.set_nodelay(!server_mode);
+    });
+    let server = axum::serve(listener, app.into_make_service());
 
     tracing::info!(
         instance = %*INSTANCE_NAME,

backend/windmill-api/src/mcp/core.rs

@@ -546,7 +546,7 @@ pub async fn setup_mcp_server(
     let service =
         StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config);
 
-    let router = Router::new().nest_service("/", service);
+    let router = Router::new().fallback_service(service);
     Ok((router, cancellation_token))
 }
 

backend/windmill-api/src/raw_apps.rs

@@ -27,7 +27,7 @@ use windmill_common::{
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
-        .route("/get_data/:version/*path", get(get_data))
+        .route("/get_data/{version}/{*path}", get(get_data))
 }
 
 #[derive(FromRow, Deserialize, Serialize)]