experiment: Add full flow executor with branch/loop support

Add flow_executor.rs that supports executing complex flows in local mode:
- ForloopFlow: iterate over arrays/ranges with sequential execution
- WhileloopFlow: execute modules while condition is true
- BranchOne: if/else branching based on conditions
- BranchAll: parallel branch execution (sequential for now)
- RawScript: inline script execution (bash, python, deno, bun)
- Identity: pass-through module

Key features:
- Uses windmill-common FlowValue types for compatibility
- Expression evaluation for input transforms with comparisons
- Proper flow status tracking with module results
- Recursive async execution with async_recursion crate

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/docker-image.yml

@@ -167,39 +167,6 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-  # disabled until we make it 100% reliable and add more meaningful tests
-  # playwright:
-  #   runs-on: [self-hosted, new]
-  #   needs: [build]
-  #   services:
-  #     postgres:
-  #       image: postgres
-  #       env:
-  #         POSTGRES_DB: windmill
-  #         POSTGRES_USER: admin
-  #         POSTGRES_PASSWORD: changeme
-  #       ports:
-  #         - 5432:5432
-  #       options: >-
-  #         --health-cmd pg_isready
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 5
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: "Docker"
-  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-  #       id: docker-container
-  #     - uses: actions/setup-node@v3
-  #       with:
-  #         node-version: 16
-  #     - name: "Playwright run"
-  #       timeout-minutes: 2
-  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
-  #     - name: "Clean up"
-  #       run: docker kill ${{ steps.docker-container.outputs.id }}
-  #       if: always()
-
   attach_amd64_binary_to_release:
     needs: [build, build_ee]
     runs-on: ubicloud

CHANGELOG.md

@@ -1,5 +1,124 @@
 # Changelog
 
+## [1.613.4](https://github.com/windmill-labs/windmill/compare/v1.613.3...v1.613.4) (2026-01-21)
+
+
+### Bug Fixes
+
+* update git sync CLI to 1.613.2 ([7848d36](https://github.com/windmill-labs/windmill/commit/7848d361a546ef8a2ef9fa20a0399f2e8473f685))
+
+## [1.613.3](https://github.com/windmill-labs/windmill/compare/v1.613.2...v1.613.3) (2026-01-21)
+
+
+### Bug Fixes
+
+* **cli:** normalize paths in wmill-lock for cross-platform compatibility ([#7645](https://github.com/windmill-labs/windmill/issues/7645)) ([8cf456d](https://github.com/windmill-labs/windmill/commit/8cf456d74c79921806edbcd0c9fde462f7202188))
+* update git sync CLI to 1.613.2 ([1e4fe01](https://github.com/windmill-labs/windmill/commit/1e4fe01293e65ac130b368fbef45a1571ee2b6d7))
+
+## [1.613.2](https://github.com/windmill-labs/windmill/compare/v1.613.1...v1.613.2) (2026-01-21)
+
+
+### Bug Fixes
+
+* azure read s3 proxy ([#7641](https://github.com/windmill-labs/windmill/issues/7641)) ([9e617a3](https://github.com/windmill-labs/windmill/commit/9e617a3979622a58ae022f9a74e2dde87a43c60e))
+* **cli:** skip branch-specific files when type is not configured ([#7643](https://github.com/windmill-labs/windmill/issues/7643)) ([287b7e7](https://github.com/windmill-labs/windmill/commit/287b7e7d971469db979e6951ee14764f6b91ed67))
+
+## [1.613.1](https://github.com/windmill-labs/windmill/compare/v1.613.0...v1.613.1) (2026-01-21)
+
+
+### Bug Fixes
+
+* fix microsoft SSO setting ([3f3df41](https://github.com/windmill-labs/windmill/commit/3f3df4163f9b6d99bc2c0f0284134b8b78f1ef6d))
+* isolate SvelteKit-specific imports for library usage ([203f678](https://github.com/windmill-labs/windmill/commit/203f6785c4ba9f7ace643259bf4e4a8f164288f3))
+
+## [1.613.0](https://github.com/windmill-labs/windmill/compare/v1.612.2...v1.613.0) (2026-01-20)
+
+
+### Features
+
+* **api:** add include_args query parameter to job list endpoints ([96dabee](https://github.com/windmill-labs/windmill/commit/96dabee22591adff5d6221e8628f7a1571b8d5a8))
+* **cli:** add workspace list command to show remote workspaces ([a08c52e](https://github.com/windmill-labs/windmill/commit/a08c52ec8f5323c645457b1dd7b32ef703fd86c4))
+* DuckDB support write to Azure ([#7618](https://github.com/windmill-labs/windmill/issues/7618)) ([73e86d9](https://github.com/windmill-labs/windmill/commit/73e86d9fc867aedb7221cf3da9df8ab573734d0f))
+* **mcp:** handle server oauth ([#7585](https://github.com/windmill-labs/windmill/issues/7585)) ([09adc58](https://github.com/windmill-labs/windmill/commit/09adc58a678da2d59d20e27a6b528b79843f122f))
+* otel REST tracing ([#7571](https://github.com/windmill-labs/windmill/issues/7571)) ([95df7b9](https://github.com/windmill-labs/windmill/commit/95df7b9a6a8ffcbca92b3249a61d97c32c9dbc4f))
+* **raw-apps:** add ctx input type for secure backend-resolved user context ([#7621](https://github.com/windmill-labs/windmill/issues/7621)) ([c143e78](https://github.com/windmill-labs/windmill/commit/c143e78d7fdd866b2e30c036ef48e907dda6ad6c))
+* **raw-apps:** add public URL and custom path support for raw apps ([#7630](https://github.com/windmill-labs/windmill/issues/7630)) ([baf060d](https://github.com/windmill-labs/windmill/commit/baf060df7474620b144e4a438274866bdfc41881))
+* **raw-apps:** enable hash-based routing with URL sync for shareable URLs ([#7624](https://github.com/windmill-labs/windmill/issues/7624)) ([3205949](https://github.com/windmill-labs/windmill/commit/32059499d5fa9aed1f8149f427732d1f0500dce5))
+
+
+### Bug Fixes
+
+* **cli:** recognize branch-specific folder files in getTypeStrFromPath ([6f35279](https://github.com/windmill-labs/windmill/commit/6f35279126b875d09af23d4618e86f87be064679))
+* **cli:** recognize branch-specific settings and encryption_key files ([5c1c682](https://github.com/windmill-labs/windmill/commit/5c1c682dcaa1a4ce80ee4de78b80c9ace395092a))
+* **frontend:** improve raw app history  ([#7625](https://github.com/windmill-labs/windmill/issues/7625)) ([687175c](https://github.com/windmill-labs/windmill/commit/687175c6a85f47c707bb429008c93ec0981c50e2))
+* **frontend:** set editor font size to the same default as text ([#7631](https://github.com/windmill-labs/windmill/issues/7631)) ([d884ddb](https://github.com/windmill-labs/windmill/commit/d884ddb7eb611f17a8e0ed41998953fb47b6cc21))
+* S3 advanced custom permissions ([#7632](https://github.com/windmill-labs/windmill/issues/7632)) ([1526d3a](https://github.com/windmill-labs/windmill/commit/1526d3ae2b3139bbfa23aef012bbe7f9b2132732))
+
+## [1.612.2](https://github.com/windmill-labs/windmill/compare/v1.612.1...v1.612.2) (2026-01-19)
+
+
+### Bug Fixes
+
+* add HIDE_WORKERS_FOR_NON_ADMINS env var and workspace-scoped custom_tags endpoint ([#7613](https://github.com/windmill-labs/windmill/issues/7613)) ([f33b799](https://github.com/windmill-labs/windmill/commit/f33b79936b8666242f2235d4fa6d4e7488123194))
+* **mcp:** fix empty args format + sanitize tool name ([#7615](https://github.com/windmill-labs/windmill/issues/7615)) ([f55dac6](https://github.com/windmill-labs/windmill/commit/f55dac69582000f0bfdae6dbed2d33f2e48087b2))
+
+## [1.612.1](https://github.com/windmill-labs/windmill/compare/v1.612.0...v1.612.1) (2026-01-19)
+
+
+### Bug Fixes
+
+* fix runs page initialization ([1438b26](https://github.com/windmill-labs/windmill/commit/1438b263102ccf22612f98e59596c7e51083df71))
+* update git sync CLI to 1.612.0 ([8daeccc](https://github.com/windmill-labs/windmill/commit/8daeccc89fc0405143a2b553520f9a42272e3c26))
+
+## [1.612.0](https://github.com/windmill-labs/windmill/compare/v1.611.0...v1.612.0) (2026-01-19)
+
+
+### Features
+
+* **cli:** add branch-specific items for folders and settings ([#7611](https://github.com/windmill-labs/windmill/issues/7611)) ([3ec9439](https://github.com/windmill-labs/windmill/commit/3ec94395dcc6a179a4d5dde3a5b88aeb1053ada3))
+* move job metrics from ee to ce ([#7608](https://github.com/windmill-labs/windmill/issues/7608)) ([c04eb37](https://github.com/windmill-labs/windmill/commit/c04eb371ccd805e8d0d0a03b4ef654c7a8131ccd))
+
+
+### Bug Fixes
+
+* **frontend:** fix centered page shift when scroll ([#7610](https://github.com/windmill-labs/windmill/issues/7610)) ([c1ec159](https://github.com/windmill-labs/windmill/commit/c1ec159471d3fabb9cb7b9023d662726a9cf1f93))
+* **frontend:** improve ai settings page ([#7606](https://github.com/windmill-labs/windmill/issues/7606)) ([9359ad8](https://github.com/windmill-labs/windmill/commit/9359ad820ded8a4a94195ee76bbe6210f6e8eb9f))
+* **frontend:** improve loading centered modal ui ([#7605](https://github.com/windmill-labs/windmill/issues/7605)) ([30da9e6](https://github.com/windmill-labs/windmill/commit/30da9e69f88ba4621bb3ee35287f914389930bb6))
+
+## [1.611.0](https://github.com/windmill-labs/windmill/compare/v1.610.1...v1.611.0) (2026-01-19)
+
+
+### Features
+
+* add HashiCorp Vault secret storage integration ([#7599](https://github.com/windmill-labs/windmill/issues/7599)) ([1b9d1c5](https://github.com/windmill-labs/windmill/commit/1b9d1c56c7e49042677326eb397e10d34a3ddcdf))
+
+
+### Bug Fixes
+
+* **flow-chat:** handle SSE timeout and fix temp message race condition ([4f8110e](https://github.com/windmill-labs/windmill/commit/4f8110eb9852b78b48aabbed114c75cbf0d1a2ef))
+
+## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17)
+
+
+### Bug Fixes
+
+* resolve BlobPart type incompatibility between Deno and Node.js ([2eac74c](https://github.com/windmill-labs/windmill/commit/2eac74cef4aa5a987fb16110388f99e912951db8))
+* use type cast instead of slice() for BlobPart compatibility ([ff77154](https://github.com/windmill-labs/windmill/commit/ff771546380ef26dfa443f9d459853048bc8029c))
+
+## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17)
+
+
+### Features
+
+* add private npm registry proxy support for ATA in webide ([#7597](https://github.com/windmill-labs/windmill/issues/7597)) ([b3cb41e](https://github.com/windmill-labs/windmill/commit/b3cb41efa4520fd3243a6dbab7c985923dd538e3))
+* add workspace success handler with 60s TTL caching ([#7598](https://github.com/windmill-labs/windmill/issues/7598)) ([73c4ce3](https://github.com/windmill-labs/windmill/commit/73c4ce30127af587510b16fbc7fa223436f6845c))
+
+
+### Bug Fixes
+
+* handle missing storage key in S3Object for write_s3_file ([03daa34](https://github.com/windmill-labs/windmill/commit/03daa341eb91118c27e92b1b51731400a0dce30c))
+* improve job deletion performance and batching ([8dd5e81](https://github.com/windmill-labs/windmill/commit/8dd5e81a32b97814eab3b72964807bdfe0ea8b49))
+
 ## [1.609.0](https://github.com/windmill-labs/windmill/compare/v1.608.0...v1.609.0) (2026-01-16)
 
 

backend/.sqlx/query-002d68d7c4437522a6dae95af007a356217bbae06b8453f0c32046f0cbf20dcb.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_by",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "002d68d7c4437522a6dae95af007a356217bbae06b8453f0c32046f0cbf20dcb"
+}

backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c"
+}

backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336"
+}

backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path FROM variable WHERE is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088"
+}

backend/.sqlx/query-085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "success_handler_extra_args",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "085bbcfcebc9e56984295046c48722f3510548eaf525c499ad0e24cdf8332b22"
+}

backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json

@@ -177,6 +177,16 @@
         "ordinal": 34,
         "name": "teams_team_guid",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 35,
+        "name": "success_handler",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 36,
+        "name": "success_handler_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -219,6 +229,8 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac.json

@@ -1,38 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\nWITH lockable_counters AS (\n    SELECT concurrency_id, job_uuids\n    FROM concurrency_counter\n    WHERE job_uuids != '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n    SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n    FROM lockable_counters\n),\norphaned_job_uuids AS (\n    SELECT job_uuid\n    FROM all_job_uuids\n    WHERE job_uuid NOT IN (\n        SELECT id::text \n        FROM v2_job_queue \n        FOR SHARE SKIP LOCKED\n    )\n),\norphaned_array AS (\n    SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n    SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n    FROM lockable_counters lc, orphaned_array oa\n    WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n    UPDATE concurrency_counter \n    SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n    FROM orphaned_array\n    WHERE concurrency_counter.concurrency_id IN (\n        SELECT concurrency_id FROM before_update\n    )\n    RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n    SELECT bu.concurrency_id, \n           bu.job_uuids AS original_job_uuids,\n           unnest(bu.orphaned_keys) AS orphaned_key\n    FROM before_update bu\n)\nSELECT \n    eo.concurrency_id,\n    eo.orphaned_key,\n    eo.original_job_uuids,\n    ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "concurrency_id",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "orphaned_key",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "original_job_uuids",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 3,
-        "name": "updated_job_uuids",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      null,
-      false,
-      false
-    ]
-  },
-  "hash": "0a1c10bd2232b0770a7816e1bd8d758dc393f797890d597e5996146247f512ac"
-}

backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "TextArray",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d"
+}

backend/.sqlx/query-10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "success_handler",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "10a40902f5aba38aea23f2b0776b54752362c9cc7deb179f75c7f1795866d13a"
+}

backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_secret",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -177,6 +177,16 @@
         "ordinal": 34,
         "name": "teams_team_guid",
         "type_info": "Text"
+      },
+      {
+        "ordinal": 35,
+        "name": "success_handler",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 36,
+        "name": "success_handler_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -219,6 +229,8 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "is_secret",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65"
+}

backend/.sqlx/query-1b4f7485c015338536d781838448c96ce686fce217be21ec15a8900b772f02a3.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_server_code\n         (code, client_id, user_email, workspace_id, scopes, redirect_uri, code_challenge, code_challenge_method)\n         VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Text",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1b4f7485c015338536d781838448c96ce686fce217be21ec15a8900b772f02a3"
+}

backend/.sqlx/query-1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1ccc9db6c6cb51bebeb90d003e12c8f7922450515d1eaee8b3abee5b31f72527"
+}

backend/.sqlx/query-1d8c2f54118b352dc13058dbb9b6e3f6ca4961b68d7e409386e655a61c54e0d0.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1d8c2f54118b352dc13058dbb9b6e3f6ca4961b68d7e409386e655a61c54e0d0"
+}

backend/.sqlx/query-2922c242228b2188b8abcda02b37d6fd220659dcd9e16d4bb110202321bc06cf.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "client_name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "redirect_uris",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "2922c242228b2188b8abcda02b37d6fd220659dcd9e16d4bb110202321bc06cf"
+}

backend/.sqlx/query-2c231a2cd267d8d6d28a22d166a50cc6b4df813a15c613eb1960eff202c517f8.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_refresh_token\n         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)\n         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2c231a2cd267d8d6d28a22d166a50cc6b4df813a15c613eb1960eff202c517f8"
+}

backend/.sqlx/query-2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        UPDATE workspace_settings\n        SET success_handler = NULL\n        WHERE workspace_id = 'test-workspace'\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "2c2ed5c91a4893966e94305df8b9f8ac904c8d0781a83fdea66c918320fb5601"
+}

backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n        VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95"
+}

backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value FROM variable\n           WHERE path LIKE ('u/' || $1 || '/%')\n           AND workspace_id = $2\n           AND is_secret = true\n           AND value LIKE '$vault:%'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e"
+}

backend/.sqlx/query-306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM v2_job_completed\n         WHERE id IN (\n             SELECT id FROM v2_job_completed\n             WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n             ORDER BY completed_at ASC\n             LIMIT $2\n             FOR UPDATE SKIP LOCKED\n         )\n         RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "306e0156ee1541710c1c6512ecb4f61baeb3ae6f31ba3fd57a3ec485108a7f49"
+}

backend/.sqlx/query-369f8ecde50af034f06d339ecef8fc55a0113b4156274d03d5af643c3da73fa4.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM mcp_oauth_server_code\n         WHERE code = $1 AND expires_at > now()\n         RETURNING code, client_id, user_email, workspace_id, scopes, redirect_uri,\n                   code_challenge, code_challenge_method",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "code",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "user_email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "scopes",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 5,
+        "name": "redirect_uri",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "code_challenge",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "code_challenge_method",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "369f8ecde50af034f06d339ecef8fc55a0113b4156274d03d5af643c3da73fa4"
+}

backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200"
+}

backend/.sqlx/query-3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        INSERT INTO group_ (workspace_id, name, summary, extra_perms)\n        VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{\"u/test-user\": true}')\n        ON CONFLICT DO NOTHING\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "3a71ab02239d8dd4e2f440a35155459a7d5091c125a0998e79d89f8c560a3fd3"
+}

backend/.sqlx/query-45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\nWITH rows_to_delete AS (\n    SELECT concurrency_id\n    FROM concurrency_counter\n    \n    WHERE job_uuids = '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete)  RETURNING concurrency_id",
+  "query": "\nWITH rows_to_delete AS (\n    SELECT concurrency_id\n    FROM concurrency_counter\n\n    WHERE job_uuids = '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n)\nDELETE FROM concurrency_counter\nWHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete)  RETURNING concurrency_id",
   "describe": {
     "columns": [
       {
@@ -16,5 +16,5 @@
       false
     ]
   },
-  "hash": "c6ef0acdf20bd71dd26de981fb49f178ba8a1b8c1e01e0fec1dfd6a54ea7a894"
+  "hash": "45d0552a169b63241a0ecc0e7ec6a533d7eec520376b99b2fd8000f0f0acba7b"
 }

backend/.sqlx/query-4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            UPDATE\n                workspace_settings\n            SET\n                success_handler = NULL,\n                success_handler_extra_args = NULL\n            WHERE\n                workspace_id = $1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4977b4027861ca234e95559a8b550bc8849f7ed60a471d46f3be3d39664a48ca"
+}

backend/.sqlx/query-4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n            COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box<str>\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n        FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n        WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n            AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n            AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n            AND q.canceled_by IS NULL\n            \n        ",
+  "query": "\n        SELECT\n            j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n            COALESCE(s.flow_status, s.workflow_as_code_status) AS \"flow_status: Box<str>\", r.ping AS last_ping, j.same_worker AS \"same_worker?\"\n        FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n        WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n            AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')\n            AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n            AND q.canceled_by IS NULL\n\n        ",
   "describe": {
     "columns": [
       {
@@ -54,5 +54,5 @@
       false
     ]
   },
-  "hash": "70a6880960d17218bc5bf05287e2a6d9a6393c6bb1783ab8903d87dd099e236b"
+  "hash": "4d80985dd0794a01a2af18ae7abf4a3ab8ba3d162ed5d04735caea7295da0b20"
 }

backend/.sqlx/query-5769af6cfc749881b3f21d42d2c79b4c3e6788ba521ef5736f46d6ec8447ad8f.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM mcp_oauth_server_code WHERE expires_at <= now() RETURNING code",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "code",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "5769af6cfc749881b3f21d42d2c79b4c3e6788ba521ef5736f46d6ec8447ad8f"
+}

backend/.sqlx/query-5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM v2_job_completed c\n                    WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval \n                    RETURNING c.id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Uuid"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10"
-}

backend/.sqlx/query-598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\nWITH lockable_counters AS (\n    SELECT concurrency_id, job_uuids\n    FROM concurrency_counter\n    WHERE job_uuids != '{}'::jsonb\n    FOR UPDATE SKIP LOCKED\n),\nall_job_uuids AS (\n    SELECT DISTINCT jsonb_object_keys(job_uuids) AS job_uuid\n    FROM lockable_counters\n),\norphaned_job_uuids AS (\n    SELECT job_uuid\n    FROM all_job_uuids\n    WHERE job_uuid NOT IN (\n        SELECT id::text\n        FROM v2_job_queue\n        FOR SHARE SKIP LOCKED\n    )\n),\norphaned_array AS (\n    SELECT ARRAY(SELECT job_uuid FROM orphaned_job_uuids) AS orphaned_keys\n),\nbefore_update AS (\n    SELECT lc.concurrency_id, lc.job_uuids, oa.orphaned_keys\n    FROM lockable_counters lc, orphaned_array oa\n    WHERE lc.job_uuids ?| oa.orphaned_keys\n),\naffected_rows AS (\n    UPDATE concurrency_counter\n    SET job_uuids = job_uuids - orphaned_array.orphaned_keys\n    FROM orphaned_array\n    WHERE concurrency_counter.concurrency_id IN (\n        SELECT concurrency_id FROM before_update\n    )\n    RETURNING concurrency_id, job_uuids AS updated_job_uuids\n),\nexpanded_orphaned AS (\n    SELECT bu.concurrency_id,\n           bu.job_uuids AS original_job_uuids,\n           unnest(bu.orphaned_keys) AS orphaned_key\n    FROM before_update bu\n)\nSELECT\n    eo.concurrency_id,\n    eo.orphaned_key,\n    eo.original_job_uuids,\n    ar.updated_job_uuids\nFROM expanded_orphaned eo\nJOIN affected_rows ar ON eo.concurrency_id = ar.concurrency_id\nWHERE eo.original_job_uuids ? eo.orphaned_key\nORDER BY eo.concurrency_id, eo.orphaned_key\n",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "concurrency_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "orphaned_key",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "original_job_uuids",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 3,
+        "name": "updated_job_uuids",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null,
+      false,
+      false
+    ]
+  },
+  "hash": "598805056a3b102c9c2fd69d8330fa78b06106236522e48ab5fe009cac059468"
+}

backend/.sqlx/query-5b5cb5339208847bd542f8d903d028803a286e69443a151cd1e3d16da7e8e4f7.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO otel_traces (\n                trace_id, span_id, trace_state, parent_span_id, flags,\n                name, kind, start_time_unix_nano, end_time_unix_nano,\n                attributes, dropped_attributes_count,\n                events, dropped_events_count,\n                links, dropped_links_count,\n                status\n            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bytea",
+        "Bytea",
+        "Text",
+        "Bytea",
+        "Int4",
+        "Text",
+        "Int4",
+        "Int8",
+        "Int8",
+        "Jsonb",
+        "Int4",
+        "Jsonb",
+        "Int4",
+        "Jsonb",
+        "Int4",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5b5cb5339208847bd542f8d903d028803a286e69443a151cd1e3d16da7e8e4f7"
+}

backend/.sqlx/query-5c9ed4d8d16c77c0c6b42e9ee211168573162745060788fbca188ed405c423cd.json

@@ -0,0 +1,89 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mcp_oauth_refresh_token\n         SET used_at = now()\n         WHERE refresh_token = $1\n           AND client_id = $2\n           AND used_at IS NULL\n           AND NOT revoked\n           AND expires_at > now()\n         RETURNING id, refresh_token, access_token, client_id, user_email, workspace_id,\n                   scopes, token_family, created_at, expires_at, used_at, revoked",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "refresh_token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "access_token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "user_email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "scopes",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 7,
+        "name": "token_family",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 8,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 9,
+        "name": "expires_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 10,
+        "name": "used_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 11,
+        "name": "revoked",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "5c9ed4d8d16c77c0c6b42e9ee211168573162745060788fbca188ed405c423cd"
+}

backend/.sqlx/query-5ffb1c49d8d001253a71c6b9bd90e58416d59a9a855afd1ec0a814937583461f.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mcp_oauth_refresh_token SET revoked = TRUE WHERE token_family = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5ffb1c49d8d001253a71c6b9bd90e58416d59a9a855afd1ec0a814937583461f"
+}

backend/.sqlx/query-6376f88654dbbd85a68c507480fc4918958244abd7a1f81f32a0e60f7e5f9464.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_server_client (client_id, client_name, redirect_uris)\n         VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "TextArray"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6376f88654dbbd85a68c507480fc4918958244abd7a1f81f32a0e60f7e5f9464"
+}

backend/.sqlx/query-66e0968fe9f757755945a7010153821cf73ace9d6692750ccc4cca37701ed77a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM token WHERE token = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "66e0968fe9f757755945a7010153821cf73ace9d6692750ccc4cca37701ed77a"
+}

backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d"
+}

backend/.sqlx/query-6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        UPDATE workspace_settings\n        SET success_handler = 'script/f/test/success_handler',\n            success_handler_extra_args = '{\"key\": \"value\"}'::json\n        WHERE workspace_id = 'test-workspace'\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "6eaa0af9c5e0a8d64c3202753c9c3cef39361b0987a22b7d2b2346652f81177e"
+}

backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT\n            id,\n            runnable_path,\n            permissioned_as_email,\n            parent_job\n        FROM v2_job\n        WHERE workspace_id = 'test-workspace'\n            AND permissioned_as_email = 'success_handler@windmill.dev'\n        ORDER BY created_at DESC\n        LIMIT 1\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "runnable_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "permissioned_as_email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "parent_job",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496"
+}

backend/.sqlx/query-7e4aa6b19b110bca423b3a3f428826d92b9808c64ef989fef2142bc8e02d6630.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)\n         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Text",
+        "TextArray",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7e4aa6b19b110bca423b3a3f428826d92b9808c64ef989fef2142bc8e02d6630"
+}

backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a"
+}

backend/.sqlx/query-9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            UPDATE\n                workspace_settings\n            SET\n                success_handler = $1,\n                success_handler_extra_args = $2\n            WHERE\n                workspace_id = $3\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Json",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9044c71dc8aab7f5d17bf27c4702e67f3eee46d7dbccb09cadeaafa0534952d6"
+}

backend/.sqlx/query-90d93fd3bd91e468c1e796e41e31e4f15a825b442346c7386836127bc5723c93.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT json_build_object(\n            'trace_id', encode(trace_id, 'hex'),           -- BYTEA to hex string\n            'span_id', encode(span_id, 'hex'),             -- BYTEA to hex string\n            'parent_span_id', encode(parent_span_id, 'hex'), -- BYTEA to hex string\n            'trace_state', trace_state,\n            'flags', flags,\n            'name', name,\n            'kind', kind,\n            'start_time_unix_nano', start_time_unix_nano,\n            'end_time_unix_nano', end_time_unix_nano,\n            'attributes', attributes,\n            'dropped_attributes_count', dropped_attributes_count,\n            'events', events,\n            'dropped_events_count', dropped_events_count,\n            'links', links,\n            'dropped_links_count', dropped_links_count,\n            'status', status\n        ) as \"span!\"\n        FROM otel_traces\n        WHERE trace_id = $1\n        ORDER BY start_time_unix_nano",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "span!",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bytea"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "90d93fd3bd91e468c1e796e41e31e4f15a825b442346c7386836127bc5723c93"
+}

backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881"
+}

backend/.sqlx/query-99398d6d6aa04235226f1a5d0f100aea034d7ee2c86aa8fa5ccec0e3560965fd.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT token_family FROM mcp_oauth_refresh_token\n                 WHERE refresh_token = $1 AND used_at IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token_family",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "99398d6d6aa04235226f1a5d0f100aea034d7ee2c86aa8fa5ccec0e3560965fd"
+}

backend/.sqlx/query-9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM concurrency_key WHERE  ended_at <= now() - ($1::bigint::text || ' s')::interval ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "9da0cea2a5d0464ca78cfeccf6cedf2b1c0e6e6cb3c9183a937a68465debdb06"
-}

backend/.sqlx/query-a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        UPDATE workspace_settings\n        SET success_handler = 'flow/f/test/success_flow'\n        WHERE workspace_id = 'test-workspace'\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "a2bc410e1066ddc89ca4557ca0e29339e41cbc7a7ef6ad97b3f0eb93de04c359"
+}

backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81"
+}

backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8"
+}

backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f"
+}

backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018"
+}

backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513"
+}

backend/.sqlx/query-d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            workspace_id,\n            slack_team_id,\n            teams_team_id,\n            teams_team_name,\n            teams_team_guid,\n            slack_name,\n            slack_command_script,\n            teams_command_script,\n            slack_email,\n            slack_oauth_client_id,\n            slack_oauth_client_secret,\n            auto_invite_domain,\n            auto_invite_operator,\n            auto_add,\n            customer_id,\n            plan,\n            webhook,\n            deploy_to,\n            ai_config,\n            error_handler,\n            error_handler_extra_args,\n            error_handler_muted_on_cancel,\n            large_file_storage,\n            datatable,\n            ducklake,\n            git_sync,\n            deploy_ui,\n            default_app,\n            default_scripts,\n            mute_critical_alerts,\n            color,\n            operator_settings,\n            git_app_installations,\n            auto_add_instance_groups,\n            auto_add_instance_groups_roles\n        FROM\n            workspace_settings\n        WHERE\n            workspace_id = $1\n        ",
+  "query": "\n        SELECT\n            workspace_id,\n            slack_team_id,\n            teams_team_id,\n            teams_team_name,\n            teams_team_guid,\n            slack_name,\n            slack_command_script,\n            teams_command_script,\n            slack_email,\n            slack_oauth_client_id,\n            slack_oauth_client_secret,\n            auto_invite_domain,\n            auto_invite_operator,\n            auto_add,\n            customer_id,\n            plan,\n            webhook,\n            deploy_to,\n            ai_config,\n            error_handler,\n            error_handler_extra_args,\n            error_handler_muted_on_cancel,\n            success_handler,\n            success_handler_extra_args,\n            large_file_storage,\n            datatable,\n            ducklake,\n            git_sync,\n            deploy_ui,\n            default_app,\n            default_scripts,\n            mute_critical_alerts,\n            color,\n            operator_settings,\n            git_app_installations,\n            auto_add_instance_groups,\n            auto_add_instance_groups_roles\n        FROM\n            workspace_settings\n        WHERE\n            workspace_id = $1\n        ",
   "describe": {
     "columns": [
       {
@@ -115,66 +115,76 @@
       },
       {
         "ordinal": 22,
+        "name": "success_handler",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 23,
+        "name": "success_handler_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 24,
         "name": "large_file_storage",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 23,
+        "ordinal": 25,
         "name": "datatable",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 24,
+        "ordinal": 26,
         "name": "ducklake",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 25,
+        "ordinal": 27,
         "name": "git_sync",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 26,
+        "ordinal": 28,
         "name": "deploy_ui",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 27,
+        "ordinal": 29,
         "name": "default_app",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 28,
+        "ordinal": 30,
         "name": "default_scripts",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 29,
+        "ordinal": 31,
         "name": "mute_critical_alerts",
         "type_info": "Bool"
       },
       {
-        "ordinal": 30,
+        "ordinal": 32,
         "name": "color",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 31,
+        "ordinal": 33,
         "name": "operator_settings",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 32,
+        "ordinal": 34,
         "name": "git_app_installations",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 33,
+        "ordinal": 35,
         "name": "auto_add_instance_groups",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 34,
+        "ordinal": 36,
         "name": "auto_add_instance_groups_roles",
         "type_info": "Jsonb"
       }
@@ -217,10 +227,12 @@
       true,
       true,
       true,
+      true,
+      true,
       false,
       true,
       true
     ]
   },
-  "hash": "3c53de373b9f1034b5f43002bf4715e12ddc641f4ca52efe0335719fa9461bb0"
+  "hash": "d0dfd217a4f53aaee7b83d5b5267f74e5ad617830dd835a45e17fd378bcbeada"
 }

backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n        VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730"
+}

backend/.sqlx/query-e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        UPDATE workspace_settings\n        SET success_handler = 'script/f/test/success_handler'\n        WHERE workspace_id = 'test-workspace'\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "e3f2459d9ac3534531d5724bb2d67be6f33f8ca02ef4768bccd32a33a70a4300"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.609.0"
+version = "1.613.4"
 authors.workspace = true
 edition.workspace = true
 
@@ -18,6 +18,7 @@ members = [
     "./windmill-indexer",
     "./windmill-macros",
     "./windmill-oauth",
+    "./windmill-local",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
     "./parsers/windmill-parser-go",
@@ -35,7 +36,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.609.0"
+version = "1.613.4"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -161,6 +162,7 @@ k8s-openapi.workspace = true
 libloading.workspace = true
 bitflags.workspace = true
 globset.workspace = true
+opentelemetry-proto.workspace = true
 
 
 [target.'cfg(windows)'.dependencies]
@@ -219,6 +221,8 @@ memchr = "2.7.4"
 axum = { version = "^0.7", features = ["multipart", "macros"] }
 headers = "^0"
 hyper = { version = "^1", features = ["full"] }
+hyper-tls = "^0.6"
+hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
 tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
 tokio-stream = { version = "0.1.17" }
 tower = "^0"
@@ -233,6 +237,7 @@ thiserror = "^2"
 anyhow = "^1"
 chrono = { version = "^0.4", features = ["serde"] }
 chrono-tz = "^0.10.1"
+derive_more = { version = "1", features = ["deref", "deref_mut"], default-features = false }
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 tracing-appender = "^0"
@@ -333,6 +338,7 @@ futures-core = "^0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
 const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] }
+const-str = "0.5"
 constant_time_eq = "0.3.1"
 dyn-iter = "0.2.0"
 rsa = "^0"
@@ -405,6 +411,7 @@ async-once-cell = "0.5.4"
 aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
 crc = "^3"
 tar = "^0"
+flate2 = "^1"
 http = "^1"
 async-stream = "^0"
 
@@ -414,6 +421,7 @@ opentelemetry_sdk = { version = "0.27.1", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
 opentelemetry-appender-tracing = "0.27.0"
 opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
+opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] }
 
 bollard = "0.18.1"
 
@@ -455,3 +463,6 @@ oracle = { version = "0.6.3", features = ["chrono"] }
 rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
 strum = { version = "0.27", features = ["derive"] }
 strum_macros = "^0"
+hudsucker = { version = "0.22", features = ["rcgen-ca", "native-tls-client"] }
+hyper-http-proxy = { version = "1", default-features = false, features = ["native-tls"] }
+rcgen = "0.13"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-da0a3751abd2b0fed46ca5c7a3aef35923403b28
+0bfcfe263622f48c086331b628c1239d5cea0b37

backend/migrations/20260112000000_otel_traces.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS otel_traces_time_idx;
+DROP INDEX IF EXISTS otel_traces_trace_time_idx;
+DROP TABLE IF EXISTS otel_traces;

backend/migrations/20260112000000_otel_traces.up.sql

@@ -0,0 +1,36 @@
+-- OpenTelemetry Span storage (all fields from proto::Span).
+-- See: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto
+
+CREATE TABLE IF NOT EXISTS otel_traces (
+    -- Identity fields (BYTEA for efficient storage and querying)
+    trace_id             BYTEA NOT NULL,              -- 16 bytes (proto: bytes)
+    span_id              BYTEA NOT NULL,              -- 8 bytes (proto: bytes)
+    trace_state          TEXT NOT NULL DEFAULT '',    -- W3C trace-context (proto: string)
+    parent_span_id       BYTEA NOT NULL DEFAULT '', -- 8 bytes, empty if root span (proto: bytes)
+    flags                INTEGER NOT NULL DEFAULT 0, -- W3C trace flags (proto: fixed32)
+    -- Core fields
+    name                 TEXT NOT NULL,              -- operation name (proto: string)
+    kind                 INTEGER NOT NULL,           -- SpanKind enum (proto: int32)
+    start_time_unix_nano BIGINT NOT NULL,            -- (proto: fixed64, postgres has no u64)
+    end_time_unix_nano   BIGINT NOT NULL,            -- (proto: fixed64, postgres has no u64)
+    -- Attributes
+    attributes                 JSONB NOT NULL DEFAULT '[]', -- (proto: repeated KeyValue)
+    dropped_attributes_count   INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Events
+    events                     JSONB NOT NULL DEFAULT '[]', -- (proto: repeated Event)
+    dropped_events_count       INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Links
+    links                      JSONB NOT NULL DEFAULT '[]', -- (proto: repeated Link)
+    dropped_links_count        INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Status
+    status               JSONB,                      -- (proto: optional Status message)
+    PRIMARY KEY (trace_id, span_id)
+);
+
+-- Query spans by trace_id, ordered by time
+CREATE INDEX IF NOT EXISTS otel_traces_trace_time_idx ON otel_traces (trace_id, start_time_unix_nano);
+
+-- Time-based cleanup (retention policy)
+CREATE INDEX IF NOT EXISTS otel_traces_time_idx ON otel_traces (start_time_unix_nano);
+
+-- trace_id = job_id.as_bytes()

backend/migrations/20260117105517_add_workspace_success_handler.down.sql

@@ -0,0 +1,4 @@
+-- Remove success_handler columns from workspace_settings
+ALTER TABLE workspace_settings
+    DROP COLUMN IF EXISTS success_handler,
+    DROP COLUMN IF EXISTS success_handler_extra_args;

backend/migrations/20260117105517_add_workspace_success_handler.up.sql

@@ -0,0 +1,4 @@
+-- Add success_handler columns to workspace_settings
+ALTER TABLE workspace_settings
+    ADD COLUMN IF NOT EXISTS success_handler TEXT,
+    ADD COLUMN IF NOT EXISTS success_handler_extra_args JSON;

backend/migrations/20260120100000_mcp_oauth_server.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS mcp_oauth_refresh_token;
+DROP TABLE IF EXISTS mcp_oauth_server_code;
+DROP TABLE IF EXISTS mcp_oauth_server_client;

backend/migrations/20260120100000_mcp_oauth_server.up.sql

@@ -0,0 +1,44 @@
+-- OAuth server: clients that have registered with Windmill to access MCP
+-- Only public clients are supported (PKCE required, no client secrets)
+CREATE TABLE mcp_oauth_server_client (
+    client_id VARCHAR(255) PRIMARY KEY,
+    client_name VARCHAR(255) NOT NULL,
+    redirect_uris TEXT[] NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- OAuth server: authorization codes (short-lived, single-use)
+CREATE TABLE mcp_oauth_server_code (
+    code VARCHAR(64) PRIMARY KEY,
+    client_id VARCHAR(255) NOT NULL REFERENCES mcp_oauth_server_client(client_id) ON DELETE CASCADE,
+    user_email VARCHAR(255) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    scopes TEXT[] NOT NULL,
+    redirect_uri TEXT NOT NULL,
+    code_challenge VARCHAR(128),  -- PKCE
+    code_challenge_method VARCHAR(10),  -- 'S256' or 'plain'
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    expires_at TIMESTAMPTZ NOT NULL DEFAULT now() + INTERVAL '10 minutes'
+);
+
+CREATE INDEX idx_mcp_oauth_server_code_expires ON mcp_oauth_server_code(expires_at);
+
+-- MCP OAuth refresh tokens for token rotation
+CREATE TABLE mcp_oauth_refresh_token (
+    id BIGSERIAL PRIMARY KEY,
+    refresh_token VARCHAR(64) NOT NULL UNIQUE,
+    access_token VARCHAR(64) NOT NULL,
+    client_id VARCHAR(255) NOT NULL REFERENCES mcp_oauth_server_client(client_id) ON DELETE CASCADE,
+    user_email VARCHAR(255) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    scopes TEXT[] NOT NULL,
+    token_family UUID NOT NULL,  -- Groups tokens from same auth flow for theft detection
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    expires_at TIMESTAMPTZ NOT NULL,
+    used_at TIMESTAMPTZ DEFAULT NULL,  -- For rotation tracking (single-use)
+    revoked BOOLEAN NOT NULL DEFAULT FALSE  -- For theft detection
+);
+
+CREATE INDEX idx_mcp_oauth_refresh_token_token ON mcp_oauth_refresh_token(refresh_token);
+CREATE INDEX idx_mcp_oauth_refresh_token_expires ON mcp_oauth_refresh_token(expires_at);
+CREATE INDEX idx_mcp_oauth_refresh_token_family ON mcp_oauth_refresh_token(token_family);

backend/migrations/20260120204751_custom_user_instance_createrole.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20260120204751_custom_user_instance_createrole.up.sql

@@ -0,0 +1,10 @@
+
+DO $$
+BEGIN
+  ALTER ROLE custom_instance_user CREATEROLE;
+EXCEPTION
+    WHEN others THEN
+        RAISE NOTICE 'Error in custom_instance_user migration: %', SQLERRM;
+        -- Continue without failing the migration
+END
+$$;

backend/parsers/windmill-parser-py-imports/src/mapping.rs

@@ -380,5 +380,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
     "taiga" => "python-taiga",
     "docx" => "python-docx",
     "vt" => "vt-py",
+    "grpc" => "grpcio",
     // Add new entry here ^
 };

backend/src/main.rs

@@ -44,7 +44,7 @@ use windmill_common::{
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
         HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
         INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
-        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, OTEL_TRACING_PROXY_SETTING,
         MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
         NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
         PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING,
@@ -99,9 +99,9 @@ use crate::monitor::{
     reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
     reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
     reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting,
-    reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting,
-    reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
-    reload_smtp_config, reload_worker_config, MonitorIteration,
+    reload_jwt_secret_setting, reload_license_key, reload_otel_tracing_proxy_setting,
+    reload_npm_config_registry_setting, reload_pip_index_url_setting, reload_retention_period_setting,
+    reload_scim_token_setting, reload_smtp_config, reload_worker_config, MonitorIteration,
 };
 
 #[cfg(feature = "parquet")]
@@ -452,6 +452,7 @@ async fn windmill_main() -> anyhow::Result<()> {
             .unwrap_or(DEFAULT_NUM_WORKERS as i32)
     };
 
+    // TODO: maybe gate behind debug_assertions?
     if num_workers > 1 && !std::env::var("WORKER_GROUP").is_ok_and(|x| x == "native") {
         println!(
             "We STRONGLY recommend using at most 1 worker per container, use at your own risks"
@@ -802,6 +803,10 @@ Windmill Community Edition {GIT_VERSION}
         #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
         let log_indexer_f = async { Ok(()) as anyhow::Result<()> };
 
+        // Resubscribe for OTEL tracing proxy before workers_f captures killpill_rx
+        #[cfg(all(feature = "private", feature = "enterprise"))]
+        let otel_killpill_rx = killpill_rx.resubscribe();
+
         let server_f = async {
             if !is_agent {
                 if let Some(db) = conn.as_sql() {
@@ -1156,6 +1161,13 @@ Windmill Community Edition {GIT_VERSION}
                                                         KEEP_JOB_DIR_SETTING => {
                                                             load_keep_job_dir(&conn).await;
                                                         },
+                                                        OTEL_TRACING_PROXY_SETTING => {
+                                                            reload_otel_tracing_proxy_setting(&conn).await;
+                                                            if worker_mode {
+                                                                tracing::info!("OTEL tracing proxy setting changed, restarting worker");
+                                                                send_delayed_killpill(&tx, 4, "OTEL tracing proxy setting change").await;
+                                                            }
+                                                        },
                                                         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
                                                             load_require_preexisting_user(&db).await;
                                                         },
@@ -1329,7 +1341,8 @@ Windmill Community Edition {GIT_VERSION}
 
                                 // update min version explicitly.
                                 // for sql connection it is the part of monitor_db.
-                                windmill_common::worker::update_min_version(conn).await;
+                                // TODO: pass worker names for min keep-alive alerts (for HTTP connection)
+                                windmill_common::min_version::update_min_version(conn, false, vec![], false).await;
                             }
                         };
                     }
@@ -1364,6 +1377,42 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
+        let otel_tracing_proxy_f = async {
+            #[cfg(all(feature = "private", feature = "enterprise"))]
+            {
+                // Start OTEL tracing proxy for HTTP request interception
+                // Only enabled when: setting is on, worker mode (not server), and single worker (to avoid race conditions)
+                if worker_mode
+                    && num_workers == 1
+                    && windmill_worker::OTEL_TRACING_PROXY_SETTINGS
+                        .read()
+                        .await
+                        .enabled
+                {
+                    if let Some(db) = conn.as_sql() {
+                        tracing::info!(
+                            "Starting OTEL tracing proxy (port will be dynamically assigned)"
+                        );
+                        if let Err(e) =
+                            windmill_worker::start_otel_tracing_proxy(db.clone(), otel_killpill_rx)
+                                .await
+                        {
+                            tracing::error!("OTEL tracing proxy error: {}", e);
+                        }
+                    }
+                } else if windmill_worker::OTEL_TRACING_PROXY_SETTINGS
+                    .read()
+                    .await
+                    .enabled
+                    && num_workers > 1
+                {
+                    tracing::warn!("OTEL tracing proxy is enabled but num_workers > 1. Disabling to avoid race conditions. Set NUM_WORKERS=1 to enable.");
+                }
+            }
+
+            Ok(()) as anyhow::Result<()>
+        };
+
         if server_mode {
             if let Some(db) = conn.as_sql() {
                 schedule_stats(&db, &HTTP_CLIENT).await;
@@ -1378,6 +1427,7 @@ Windmill Community Edition {GIT_VERSION}
                 monitor_f,
                 server_f,
                 metrics_f,
+                otel_tracing_proxy_f,
                 indexer_f,
                 log_indexer_f
             )?;

backend/src/monitor.rs

@@ -53,7 +53,7 @@ use windmill_common::{
         HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
         JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
         LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
+        OTEL_TRACING_PROXY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
         POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
@@ -81,10 +81,10 @@ use windmill_common::{
 use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING};
 use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
 use windmill_worker::{
-    handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
-    INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN,
-    NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT,
-    POWERSHELL_REPO_URL,
+    handle_job_error, JobCompletedSender, OtelTracingProxySettings, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
+    INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS,
+    OTEL_TRACING_PROXY_SETTINGS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL,
+    PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL,
 };
 
 #[cfg(feature = "parquet")]
@@ -155,6 +155,20 @@ lazy_static::lazy_static! {
 
     .ok()
     .and_then(|x| x.parse::<u64>().ok());
+
+    /// Batch size for job cleanup deletion queries. Default: 10000.
+    /// Larger values delete more jobs per batch but hold locks longer.
+    static ref JOB_CLEANUP_BATCH_SIZE: i64 = std::env::var("JOB_CLEANUP_BATCH_SIZE")
+        .ok()
+        .and_then(|x| x.parse::<i64>().ok())
+        .unwrap_or(20000);
+
+    /// Maximum number of batches to process per cleanup iteration. Default: 10.
+    /// Set to 0 for unlimited (process until no expired jobs remain).
+    static ref JOB_CLEANUP_MAX_BATCHES: i32 = std::env::var("JOB_CLEANUP_MAX_BATCHES")
+        .ok()
+        .and_then(|x| x.parse::<i32>().ok())
+        .unwrap_or(20);
 }
 
 pub async fn initial_load(
@@ -306,6 +320,7 @@ pub async fn initial_load(
         reload_maven_repos_setting(&conn).await;
         reload_no_default_maven_setting(&conn).await;
         reload_ruby_repos_setting(&conn).await;
+        reload_otel_tracing_proxy_setting(&conn).await;
     }
 }
 
@@ -764,6 +779,35 @@ pub async fn load_keep_job_dir(conn: &Connection) {
     };
 }
 
+pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) {
+    match load_value_from_global_settings_with_conn(conn, OTEL_TRACING_PROXY_SETTING, true).await {
+        Ok(Some(settings)) => {
+            match serde_json::from_value::<OtelTracingProxySettings>(settings) {
+                Ok(new_settings) => {
+                    let mut current = OTEL_TRACING_PROXY_SETTINGS.write().await;
+                    if current.enabled != new_settings.enabled
+                        || current.enabled_languages != new_settings.enabled_languages
+                    {
+                        tracing::info!(
+                            "OTEL tracing proxy settings changed: enabled={}, languages={:?}",
+                            new_settings.enabled,
+                            new_settings.enabled_languages
+                        );
+                        *current = new_settings;
+                    }
+                }
+                Err(e) => {
+                    tracing::error!("Error parsing OTEL tracing proxy settings: {e:#}");
+                }
+            }
+        }
+        Err(e) => {
+            tracing::error!("Error loading OTEL tracing proxy setting: {e:#}");
+        }
+        _ => (),
+    };
+}
+
 pub async fn load_require_preexisting_user(db: &DB) {
     let value =
         load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await;
@@ -815,6 +859,22 @@ pub async fn delete_expired_items(db: &DB) -> () {
         Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
     }
 
+    // Clean up expired MCP OAuth refresh tokens
+    let mcp_refresh_tokens_r: std::result::Result<Vec<i64>, _> = sqlx::query_scalar(
+        "DELETE FROM mcp_oauth_refresh_token WHERE expires_at <= now() RETURNING id",
+    )
+    .fetch_all(db)
+    .await;
+
+    match mcp_refresh_tokens_r {
+        Ok(ids) => {
+            if ids.len() > 0 {
+                tracing::info!("deleted {} expired MCP OAuth refresh tokens", ids.len())
+            }
+        }
+        Err(e) => tracing::error!("Error deleting MCP OAuth refresh tokens: {}", e.to_string()),
+    }
+
     let deleted_cache = sqlx::query_scalar!(
             "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
         )
@@ -908,99 +968,170 @@ pub async fn delete_expired_items(db: &DB) -> () {
         Err(e) => tracing::error!("Error deleting expired blacklisted agent tokens: {:?}", e),
     }
 
-    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
-    if job_retention_secs > 0 {
-        match db.begin().await {
-            Ok(mut tx) => {
-                let deleted_jobs = sqlx::query_scalar!(
-                    "DELETE FROM v2_job_completed c
-                    WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval 
-                    RETURNING c.id",
-                    job_retention_secs
-                )
-                .fetch_all(&mut *tx)
-                .await;
-
-                match deleted_jobs {
-                    Ok(deleted_jobs) => {
-                        if deleted_jobs.len() > 0 {
-                            tracing::info!(
-                                "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
-                                deleted_jobs.len(),
-                                job_retention_secs,
-                                deleted_jobs,
-                            );
-                            if let Err(e) = sqlx::query!(
-                                "DELETE FROM job_stats WHERE job_id = ANY($1)",
-                                &deleted_jobs
-                            )
-                            .execute(&mut *tx)
-                            .await
-                            {
-                                tracing::error!("Error deleting job stats: {:?}", e);
-                            }
-                            match sqlx::query_scalar!(
-                                "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
-                                &deleted_jobs
-                            )
-                            .fetch_all(&mut *tx)
-                            .await
-                            {
-                                Ok(log_file_index) => {
-                                    let paths = log_file_index
-                                        .into_iter()
-                                        .filter_map(|opt| opt)
-                                        .flat_map(|inner_vec| inner_vec.into_iter())
-                                        .collect();
-                                    delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await;
-                                }
-                                Err(e) => tracing::error!("Error deleting job stats: {:?}", e),
-                            }
-                            if let Err(e) = sqlx::query!(
-                                "DELETE FROM concurrency_key WHERE  ended_at <= now() - ($1::bigint::text || ' s')::interval ",
-                                job_retention_secs
-                            )
-                            .execute(&mut *tx)
-                            .await
-                            {
-                                tracing::error!("Error deleting  custom concurrency key: {:?}", e);
-                            }
-
-                            if let Err(e) =
-                                sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs)
-                                    .execute(&mut *tx)
-                                    .await
-                            {
-                                tracing::error!("Error deleting job: {:?}", e);
-                            }
-
-                            // should already be deleted but just in case
-                            if let Err(e) = sqlx::query!(
-                                "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
-                                &deleted_jobs
-                            )
-                            .execute(&mut *tx)
-                            .await
-                            {
-                                tracing::error!("Error deleting job result stream: {:?}", e);
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        tracing::error!("Error deleting expired jobs: {:?}", e)
-                    }
-                }
-
-                match tx.commit().await {
-                    Ok(_) => (),
-                    Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err),
-                }
-            }
-            Err(err) => {
-                tracing::error!("Error deleting expired jobs: {:?}", err)
+    match sqlx::query_scalar!(
+        "DELETE FROM mcp_oauth_server_code WHERE expires_at <= now() RETURNING code",
+    )
+    .fetch_all(db)
+    .await
+    {
+        Ok(deleted_codes) => {
+            if deleted_codes.len() > 0 {
+                tracing::info!(
+                    "deleted {} expired MCP OAuth authorization codes",
+                    deleted_codes.len()
+                );
             }
         }
+        Err(e) => tracing::error!("Error deleting expired MCP OAuth authorization codes: {:?}", e),
     }
+
+    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
+    if job_retention_secs > 0 {
+        let batch_size = *JOB_CLEANUP_BATCH_SIZE;
+        let max_batches = *JOB_CLEANUP_MAX_BATCHES;
+        let cleanup_start = Instant::now();
+        let mut total_deleted = 0u64;
+        let mut batch_num = 0i32;
+
+        // Process batches until no more expired jobs or max batches reached
+        loop {
+            if max_batches > 0 && batch_num >= max_batches {
+                tracing::debug!(
+                    "Job cleanup: reached max batches limit ({}), will continue next iteration",
+                    max_batches
+                );
+                break;
+            }
+
+            // Each batch runs in its own transaction to avoid long-running locks
+            let batch_result = delete_expired_jobs_batch(db, job_retention_secs, batch_size).await;
+
+            match batch_result {
+                Ok(deleted_count) => {
+                    if deleted_count == 0 {
+                        // No more expired jobs to delete
+                        break;
+                    }
+                    total_deleted += deleted_count as u64;
+                    batch_num += 1;
+                }
+                Err(e) => {
+                    tracing::error!("Error in job cleanup batch {}: {:?}", batch_num, e);
+                    break;
+                }
+            }
+        }
+
+        if total_deleted > 0 {
+            tracing::info!(
+                "Job cleanup completed: deleted {} jobs in {} batches, took {:?}",
+                total_deleted,
+                batch_num,
+                cleanup_start.elapsed()
+            );
+        }
+
+        // Clean up concurrency keys separately (not tied to specific job IDs)
+        if let Err(e) = sqlx::query!(
+            "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
+            job_retention_secs
+        )
+        .execute(db)
+        .await
+        {
+            tracing::error!("Error deleting custom concurrency key: {:?}", e);
+        }
+    }
+}
+
+/// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments.
+/// Uses a single transaction per batch to minimize lock duration.
+/// Returns the number of jobs deleted in this batch.
+async fn delete_expired_jobs_batch(
+    db: &DB,
+    job_retention_secs: i64,
+    batch_size: i64,
+) -> error::Result<usize> {
+    let mut tx = db.begin().await?;
+
+    // Use FOR UPDATE SKIP LOCKED to avoid contention between replicas
+    // ORDER BY completed_at ensures we delete oldest jobs first
+    let deleted_jobs: Vec<Uuid> = sqlx::query_scalar!(
+        "DELETE FROM v2_job_completed
+         WHERE id IN (
+             SELECT id FROM v2_job_completed
+             WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
+             ORDER BY completed_at ASC
+             LIMIT $2
+             FOR UPDATE SKIP LOCKED
+         )
+         RETURNING id",
+        job_retention_secs,
+        batch_size
+    )
+    .fetch_all(&mut *tx)
+    .await?;
+
+    let deleted_count = deleted_jobs.len();
+
+    if deleted_count > 0 {
+        tracing::debug!(
+            "Deleting batch of {} expired jobs (retention: {}s)",
+            deleted_count,
+            job_retention_secs
+        );
+
+        // Delete related records for this batch
+        if let Err(e) = sqlx::query!(
+            "DELETE FROM job_stats WHERE job_id = ANY($1)",
+            &deleted_jobs
+        )
+        .execute(&mut *tx)
+        .await
+        {
+            tracing::error!("Error deleting job stats: {:?}", e);
+        }
+
+        match sqlx::query_scalar!(
+            "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
+            &deleted_jobs
+        )
+        .fetch_all(&mut *tx)
+        .await
+        {
+            Ok(log_file_index) => {
+                let paths = log_file_index
+                    .into_iter()
+                    .filter_map(|opt| opt)
+                    .flat_map(|inner_vec| inner_vec.into_iter())
+                    .collect();
+                delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await;
+            }
+            Err(e) => tracing::error!("Error deleting job logs: {:?}", e),
+        }
+
+        if let Err(e) = sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs)
+            .execute(&mut *tx)
+            .await
+        {
+            tracing::error!("Error deleting job: {:?}", e);
+        }
+
+        // Should already be deleted but just in case
+        if let Err(e) = sqlx::query!(
+            "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
+            &deleted_jobs
+        )
+        .execute(&mut *tx)
+        .await
+        {
+            tracing::error!("Error deleting job result stream: {:?}", e);
+        }
+    }
+
+    tx.commit().await?;
+
+    Ok(deleted_count)
 }
 
 async fn delete_log_files_from_disk_and_store(
@@ -1732,7 +1863,7 @@ pub async fn monitor_db(
 
     let update_min_worker_version_f = async {
         #[cfg(not(feature = "test_job_debouncing"))]
-        windmill_common::worker::update_min_version(conn).await;
+        windmill_common::min_version::update_min_version(conn, _worker_mode, WORKERS_NAMES.read().await.clone(), initial_load).await;
     };
 
     join!(
@@ -2262,7 +2393,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
         sqlx::query_scalar!("SELECT j.id
              FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)
              WHERE r.ping < now() - ($1 || ' seconds')::interval
-             AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false", 
+             AND q.running = true AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') AND j.same_worker = false",
              ZOMBIE_JOB_TIMEOUT.as_str())
         .fetch_all(db)
         .await
@@ -2396,8 +2527,8 @@ orphaned_job_uuids AS (
     SELECT job_uuid
     FROM all_job_uuids
     WHERE job_uuid NOT IN (
-        SELECT id::text 
-        FROM v2_job_queue 
+        SELECT id::text
+        FROM v2_job_queue
         FOR SHARE SKIP LOCKED
     )
 ),
@@ -2410,7 +2541,7 @@ before_update AS (
     WHERE lc.job_uuids ?| oa.orphaned_keys
 ),
 affected_rows AS (
-    UPDATE concurrency_counter 
+    UPDATE concurrency_counter
     SET job_uuids = job_uuids - orphaned_array.orphaned_keys
     FROM orphaned_array
     WHERE concurrency_counter.concurrency_id IN (
@@ -2419,12 +2550,12 @@ affected_rows AS (
     RETURNING concurrency_id, job_uuids AS updated_job_uuids
 ),
 expanded_orphaned AS (
-    SELECT bu.concurrency_id, 
+    SELECT bu.concurrency_id,
            bu.job_uuids AS original_job_uuids,
            unnest(bu.orphaned_keys) AS orphaned_key
     FROM before_update bu
 )
-SELECT 
+SELECT
     eo.concurrency_id,
     eo.orphaned_key,
     eo.original_job_uuids,
@@ -2453,7 +2584,7 @@ async fn cleanup_concurrency_counters_empty_keys(db: &DB) -> error::Result<()> {
 WITH rows_to_delete AS (
     SELECT concurrency_id
     FROM concurrency_counter
-    
+
     WHERE job_uuids = '{}'::jsonb
     FOR UPDATE SKIP LOCKED
 )
@@ -2489,7 +2620,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
             AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode')
             AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval
             AND q.canceled_by IS NULL
-            
+
         "#,
         FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str()
     )
@@ -2820,8 +2951,8 @@ RETURNING key,job_id
 
 async fn cleanup_debounce_keys_for_completed_jobs(db: &DB) -> error::Result<()> {
     // If min version doesn't support runnable settings, clean up debounce keys for completed jobs
-    if !*windmill_common::worker::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
-        .read()
+    if !windmill_common::min_version::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
+        .met()
         .await
     {
         let result = sqlx::query!(

backend/summarized_schema.txt

@@ -1065,6 +1065,8 @@ TABLE: workspace_settings
   - ai_config (jsonb)
   - error_handler_extra_args (json)
   - error_handler_muted_on_cancel (boolean)
+  - success_handler (text)
+  - success_handler_extra_args (json)
   - large_file_storage (jsonb)
   - git_sync (jsonb)
   - default_app (character)

backend/tests/common/mod.rs

@@ -1,12 +1,11 @@
 #![allow(dead_code)]
 
-use std::{future::Future, str::FromStr, sync::Arc};
+use std::{future::Future, str::FromStr};
 
 use futures::Stream;
 use serde::Serialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, Pool, Postgres};
-use tokio::sync::RwLock;
 use uuid::Uuid;
 use windmill_api_client::types::NewScript;
 #[cfg(feature = "python")]
@@ -498,11 +497,11 @@ pub async fn initialize_tracing() {
 }
 
 pub async fn test_for_versions<F: Future<Output = ()>>(
-    version_flags: impl Iterator<Item = Arc<RwLock<bool>>>,
+    constraints: impl Iterator<Item = &'static windmill_common::min_version::VersionConstraint>,
     test: impl Fn() -> F,
 ) {
-    for version_flag in version_flags {
-        *version_flag.write().await = true;
+    for constraint in constraints {
+        *windmill_common::min_version::MIN_VERSION.write().await = constraint.version().clone();
         test().await;
     }
 }

backend/tests/fixtures/secret_backend.sql

@@ -0,0 +1,31 @@
+-- Fixture for secret backend migration tests
+-- Sets up test secrets in the variable table
+
+-- Create a second workspace for testing workspace isolation
+INSERT INTO workspace (id, name, owner)
+VALUES ('test-workspace-2', 'test-workspace-2', 'test-user')
+ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_settings (workspace_id)
+VALUES ('test-workspace-2')
+ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_key(workspace_id, kind, key)
+VALUES ('test-workspace-2', 'cloud', 'test-key-2')
+ON CONFLICT DO NOTHING;
+
+-- Insert test secrets for workspace 1
+-- Note: The 'value' column stores encrypted values in production,
+-- but for tests we'll use plain text that the migration will handle
+INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
+VALUES
+    ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'),
+    ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'),
+    ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}')
+ON CONFLICT DO NOTHING;
+
+-- Insert test secrets for workspace 2 (to test isolation)
+INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
+VALUES
+    ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}')
+ON CONFLICT DO NOTHING;

backend/tests/job_payload.rs

@@ -3,16 +3,15 @@ mod common;
 mod job_payload {
     use serde_json::json;
     use sqlx::{Pool, Postgres};
-    use std::sync::Arc;
-    use tokio::sync::RwLock;
-    use windmill_common::flow_status::RestartedFrom;
     use windmill_common::flows::{FlowModule, FlowModuleValue, FlowValue};
     use windmill_common::jobs::JobPayload;
     use windmill_common::scripts::{ScriptHash, ScriptLang};
+    use windmill_common::flow_status::RestartedFrom;
 
     use crate::common::*;
-    use windmill_common::worker::{
-        MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440,
+    use windmill_common::min_version::{
+        MIN_VERSION, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432,
+        MIN_VERSION_IS_AT_LEAST_1_440,
     };
 
     pub async fn initialize_tracing() {
@@ -28,18 +27,15 @@ mod job_payload {
         });
     }
 
-    use lazy_static::lazy_static;
-
     use windmill_common::cache;
     use windmill_common::flows::FlowNodeId;
+    use windmill_common::min_version::VersionConstraint;
 
-    lazy_static! {
-        static ref VERSION_FLAGS: [Arc<RwLock<bool>>; 3] = [
-            MIN_VERSION_IS_AT_LEAST_1_427.clone(),
-            MIN_VERSION_IS_AT_LEAST_1_432.clone(),
-            MIN_VERSION_IS_AT_LEAST_1_440.clone(),
-        ];
-    }
+    const VERSION_FLAGS: [&VersionConstraint; 3] = [
+        &MIN_VERSION_IS_AT_LEAST_1_427,
+        &MIN_VERSION_IS_AT_LEAST_1_432,
+        &MIN_VERSION_IS_AT_LEAST_1_440,
+    ];
 
     #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
@@ -71,7 +67,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello foo!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -116,7 +112,7 @@ mod job_payload {
                 .unwrap();
             assert_eq!(job.preprocessed, Some(true));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -183,7 +179,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello foo!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         let test = || async {
             let result = RunJob::from(JobPayload::FlowScript {
                 id: flow_scripts[1],
@@ -206,7 +202,7 @@ mod job_payload {
                 json!("Did you just say \"You know nothing Jean Neige\"??!")
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -252,7 +248,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Did you just say \"Hello tests!\"??!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -282,19 +278,19 @@ mod job_payload {
 
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_427(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_427.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_427.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_432(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_432.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_432.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_440(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_440.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_440.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
@@ -323,7 +319,7 @@ mod job_payload {
                 &json!("Successful lock file generation")
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -383,7 +379,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello Jean Neige!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -414,7 +410,7 @@ mod job_payload {
                 json!({ "lock": "", "status": "Successful lock file generation" })
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -447,7 +443,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_nodes_flow".to_string(),
@@ -460,7 +456,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -520,7 +516,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_preprocessor".to_string(),
@@ -533,7 +529,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -577,7 +573,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_nodes_flow".to_string(),
@@ -590,7 +586,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -638,7 +634,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello Jean Neige!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 

backend/tests/list_jobs.rs

@@ -0,0 +1,900 @@
+use serde::Deserialize;
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+
+use windmill_common::{
+    jobs::{JobPayload, RawCode},
+    scripts::ScriptLang,
+};
+
+mod common;
+use common::*;
+
+#[derive(Debug, Deserialize)]
+struct ListJobsResponse {
+    #[serde(rename = "type")]
+    #[allow(dead_code)]
+    typ: String,
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Response struct for queue/list endpoint (no type field)
+#[derive(Debug, Deserialize)]
+struct QueueJobResponse {
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Response struct for completed/list endpoint
+#[cfg(feature = "python")]
+#[derive(Debug, Deserialize)]
+struct CompletedJobResponse {
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Test that list_jobs returns jobs without args by default
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call list_jobs without include_args
+    let response = client
+        .client()
+        .get(format!("{}/w/test-workspace/jobs/list", client.baseurl()))
+        .send()
+        .await?;
+
+    assert!(response.status().is_success(), "list_jobs should succeed");
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns jobs with args when include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns completed jobs with args when include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_completed_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included for completed jobs when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value for completed jobs"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns both queued and completed jobs with args
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_mixed_queue_and_completed(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion first
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(completed_arg): return completed_arg".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("completed_arg", json!("completed_value"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let completed_job_id = completed_job.id;
+
+    // Push another job to the queue (don't run it)
+    let queued_job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(queued_arg): return queued_arg".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("queued_arg", json!("queued_value"))
+    .push(&db)
+    .await;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the completed job
+    let completed = jobs
+        .iter()
+        .find(|j| j.id == completed_job_id.to_string())
+        .expect("should find the completed job");
+
+    assert!(
+        completed.args.is_some(),
+        "completed job should have args when include_args=true"
+    );
+    assert_eq!(
+        completed.args.as_ref().unwrap().get("completed_arg"),
+        Some(&json!("completed_value")),
+        "completed job should have correct args"
+    );
+
+    // Find the queued job
+    let queued = jobs
+        .iter()
+        .find(|j| j.id == queued_job_id.to_string())
+        .expect("should find the queued job");
+
+    assert!(
+        queued.args.is_some(),
+        "queued job should have args when include_args=true"
+    );
+    assert_eq!(
+        queued.args.as_ref().unwrap().get("queued_arg"),
+        Some(&json!("queued_value")),
+        "queued job should have correct args"
+    );
+
+    Ok(())
+}
+
+/// Test list_jobs with multiple queued jobs and include_args
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_multiple_queued_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push two jobs with different args
+    let job1_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(1))
+    .push(&db)
+    .await;
+
+    let job2_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(y): return y".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("y", json!(2))
+    .push(&db)
+    .await;
+
+    // Test with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(response.status().is_success(), "list_jobs should succeed");
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find both jobs
+    let job1 = jobs.iter().find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs.iter().find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("x"),
+        Some(&json!(1)),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("y"),
+        Some(&json!(2)),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}
+
+// ============================================================================
+// Tests for /queue/list endpoint
+// ============================================================================
+
+/// Test that queue/list returns jobs without args by default
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call queue/list without include_args
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that queue/list returns jobs with args when include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call queue/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list with include_args should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test queue/list with multiple jobs and include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_multiple_jobs_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push two jobs with different args
+    let job1_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(a): return a".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("a", json!("value_a"))
+    .push(&db)
+    .await;
+
+    let job2_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(b): return b".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("b", json!("value_b"))
+    .push(&db)
+    .await;
+
+    // Call queue/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find both jobs
+    let job1 = jobs
+        .iter()
+        .find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs
+        .iter()
+        .find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("a"),
+        Some(&json!("value_a")),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("b"),
+        Some(&json!("value_b")),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}
+
+// ============================================================================
+// Tests for /completed/list endpoint
+// ============================================================================
+
+/// Test that completed/list returns jobs without args by default
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call completed/list without include_args
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that completed/list returns jobs with args when include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call completed/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list with include_args should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test completed/list with multiple jobs and include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_multiple_jobs_with_include_args(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run first job to completion
+    let completed_job1 = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(a): return a".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("a", json!("completed_a"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job1_id = completed_job1.id;
+
+    // Run second job to completion
+    let completed_job2 = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(b): return b".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("b", json!("completed_b"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job2_id = completed_job2.id;
+
+    // Call completed/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find both completed jobs
+    let job1 = jobs
+        .iter()
+        .find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs
+        .iter()
+        .find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("a"),
+        Some(&json!("completed_a")),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("b"),
+        Some(&json!("completed_b")),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}

backend/tests/secret_backend_integration.rs

@@ -0,0 +1,557 @@
+//! Integration tests for HashiCorp Vault secret backend.
+//!
+//! These tests require:
+//! 1. A PostgreSQL database (handled by sqlx test framework)
+//! 2. A running HashiCorp Vault instance
+//! 3. The RUN_VAULT_TESTS=1 environment variable to be set
+//!
+//! Environment variables:
+//! - RUN_VAULT_TESTS=1  - Required to run the tests
+//! - VAULT_ADDR         - Vault server address (default: http://127.0.0.1:8200)
+//! - VAULT_TOKEN        - Static token for static token tests (default: test-root-token)
+//! - BASE_URL           - Windmill instance URL for JWT tests (default: http://localhost:8000)
+//!
+//! Run tests (static token mode):
+//! ```bash
+//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \
+//!     secret_backend_integration --features private,enterprise -- --nocapture
+//! ```
+//!
+//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint):
+//! ```bash
+//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \
+//!     secret_backend_integration --features private,enterprise,openidconnect -- --nocapture
+//! ```
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+mod tests {
+    use sqlx::{Pool, Postgres};
+    use std::collections::HashMap;
+    use windmill_common::secret_backend::{
+        migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection,
+        SecretBackend, VaultBackend, VaultSettings,
+    };
+
+    /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var)
+    fn should_run_vault_tests() -> bool {
+        std::env::var("RUN_VAULT_TESTS")
+            .map(|v| v == "1" || v.to_lowercase() == "true")
+            .unwrap_or(false)
+    }
+
+    /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation)
+    async fn setup_base_url() {
+        let base_url = std::env::var("BASE_URL")
+            .unwrap_or_else(|_| "http://localhost:8000".to_string());
+        let mut url = windmill_common::BASE_URL.write().await;
+        *url = base_url;
+    }
+
+    /// Skip test if RUN_VAULT_TESTS is not set
+    macro_rules! skip_if_no_vault {
+        () => {
+            if !should_run_vault_tests() {
+                println!("Skipping test: RUN_VAULT_TESTS=1 not set");
+                println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ...");
+                return;
+            }
+        };
+    }
+
+    fn vault_settings_static_token() -> VaultSettings {
+        VaultSettings {
+            address: std::env::var("VAULT_ADDR")
+                .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+            mount_path: "windmill".to_string(),
+            jwt_role: None, // Static token mode
+            namespace: None,
+            token: Some(
+                std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
+            ),
+        }
+    }
+
+    fn vault_settings_jwt() -> VaultSettings {
+        VaultSettings {
+            address: std::env::var("VAULT_ADDR")
+                .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+            mount_path: "windmill".to_string(),
+            jwt_role: Some("windmill-secrets".to_string()), // JWT mode
+            namespace: None,
+            token: None, // No static token - use JWT
+        }
+    }
+
+    // ==================== Static Token Tests ====================
+
+    /// Test Vault connection with static token
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_connection_static_token(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        println!("Testing Vault connection with static token...");
+        println!("  Address: {}", settings.address);
+
+        let result = test_vault_connection(&settings, Some(&db)).await;
+        assert!(
+            result.is_ok(),
+            "Failed to connect to Vault: {:?}",
+            result.err()
+        );
+        println!("✓ Successfully connected to Vault with static token");
+    }
+
+    /// Test basic CRUD operations with static token
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_crud_static_token(_db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-crud-static";
+        let path = "test-secret";
+        let value = "my-super-secret-value-123";
+
+        println!("Testing CRUD with static token...");
+
+        // Create
+        println!("  Creating secret...");
+        backend
+            .set_secret(workspace_id, path, value)
+            .await
+            .expect("Failed to create secret");
+        println!("  ✓ Created");
+
+        // Read
+        println!("  Reading secret...");
+        let read_value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read secret");
+        assert_eq!(read_value, value);
+        println!("  ✓ Read (value matches)");
+
+        // Update
+        println!("  Updating secret...");
+        let new_value = "updated-secret-value-456";
+        backend
+            .set_secret(workspace_id, path, new_value)
+            .await
+            .expect("Failed to update secret");
+        let updated = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read updated secret");
+        assert_eq!(updated, new_value);
+        println!("  ✓ Updated");
+
+        // Delete
+        println!("  Deleting secret...");
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete secret");
+        let result = backend.get_secret(workspace_id, path).await;
+        assert!(result.is_err(), "Secret should be deleted");
+        println!("  ✓ Deleted");
+
+        println!("✓ CRUD operations successful with static token");
+    }
+
+    // ==================== JWT Auth Tests ====================
+
+    /// Test Vault connection with JWT authentication
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_connection_jwt(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        setup_base_url().await;
+
+        let settings = vault_settings_jwt();
+        println!("Testing Vault connection with JWT auth...");
+        println!("  Address: {}", settings.address);
+        println!("  JWT Role: {:?}", settings.jwt_role);
+        println!("  BASE_URL: {}", windmill_common::BASE_URL.read().await.clone());
+
+        let result = test_vault_connection(&settings, Some(&db)).await;
+        assert!(
+            result.is_ok(),
+            "Failed to connect to Vault with JWT: {:?}",
+            result.err()
+        );
+        println!("✓ Successfully connected to Vault with JWT auth");
+    }
+
+    /// Test basic CRUD operations with JWT authentication
+    #[cfg(feature = "openidconnect")]
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_crud_jwt(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        setup_base_url().await;
+
+        let settings = vault_settings_jwt();
+        let backend = VaultBackend::new_with_db(settings, db.clone());
+
+        let workspace_id = "test-crud-jwt";
+        let path = "jwt-test-secret";
+        let value = "jwt-authenticated-secret-value";
+
+        println!("Testing CRUD with JWT auth...");
+
+        // Create
+        println!("  Creating secret...");
+        backend
+            .set_secret(workspace_id, path, value)
+            .await
+            .expect("Failed to create secret with JWT");
+        println!("  ✓ Created");
+
+        // Read
+        println!("  Reading secret...");
+        let read_value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read secret with JWT");
+        assert_eq!(read_value, value);
+        println!("  ✓ Read (value matches)");
+
+        // Delete (cleanup)
+        println!("  Deleting secret...");
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete secret with JWT");
+        println!("  ✓ Deleted");
+
+        println!("✓ CRUD operations successful with JWT auth");
+    }
+
+    // ==================== Migration Tests ====================
+
+    /// Test migration from database to Vault
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        // Verify Vault connection
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // Check initial state
+        let secrets_before = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query secrets");
+
+        println!(
+            "Found {} secrets in database before migration:",
+            secrets_before.len()
+        );
+        for s in &secrets_before {
+            println!("  - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+        }
+
+        // Run migration
+        println!("\nMigrating secrets to Vault...");
+        let report = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration to Vault failed");
+
+        println!("Migration report:");
+        println!("  Total secrets: {}", report.total_secrets);
+        println!("  Migrated: {}", report.migrated_count);
+        println!("  Failed: {}", report.failed_count);
+
+        if !report.failures.is_empty() {
+            println!("  Failures:");
+            for f in &report.failures {
+                println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+            }
+        }
+
+        assert_eq!(report.failed_count, 0, "Migration had failures");
+        assert!(report.migrated_count > 0, "No secrets were migrated");
+
+        // Verify secrets in Vault
+        println!("\nVerifying secrets in Vault...");
+        let vault_backend = VaultBackend::new(settings.clone());
+
+        for secret in &secrets_before {
+            let result = vault_backend
+                .get_secret(&secret.workspace_id, &secret.path)
+                .await;
+            assert!(
+                result.is_ok(),
+                "Failed to read secret {}/{} from Vault: {:?}",
+                secret.workspace_id,
+                secret.path,
+                result.err()
+            );
+            println!(
+                "  ✓ {}/{} exists in Vault",
+                secret.workspace_id, secret.path
+            );
+        }
+
+        println!("\n✓ Migration to Vault completed successfully");
+    }
+
+    /// Test migration from Vault back to database
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // First migrate TO Vault
+        println!("Setting up: migrating secrets to Vault first...");
+        let to_vault = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Initial migration to Vault failed");
+        assert!(to_vault.migrated_count > 0, "No secrets to test with");
+        println!("  Migrated {} secrets to Vault", to_vault.migrated_count);
+
+        // Clear database values
+        println!("\nClearing database secret values...");
+        sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
+            .execute(&db)
+            .await
+            .expect("Failed to clear values");
+
+        // Migrate back from Vault
+        println!("\nMigrating secrets from Vault to database...");
+        let report = migrate_secrets_to_database(&db, &settings)
+            .await
+            .expect("Migration to database failed");
+
+        println!("Migration report:");
+        println!("  Total secrets: {}", report.total_secrets);
+        println!("  Migrated: {}", report.migrated_count);
+        println!("  Failed: {}", report.failed_count);
+
+        assert_eq!(report.failed_count, 0, "Migration had failures");
+        assert!(report.migrated_count > 0, "No secrets were migrated");
+
+        // Verify restored
+        let restored = sqlx::query!(
+            "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'"
+        )
+        .fetch_one(&db)
+        .await
+        .expect("Failed to count restored");
+
+        assert!(
+            restored.count.unwrap_or(0) > 0,
+            "No secrets were restored in database"
+        );
+
+        println!("\n✓ Migration to database completed successfully");
+    }
+
+    // ==================== Variable Rename Tests ====================
+
+    /// Test renaming a variable path in Vault
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_variable_rename(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        let _ = &db; // suppress unused warning
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-workspace";
+        let old_path = "u/test-user/old_secret_name";
+        let new_path = "u/test-user/new_secret_name";
+        let value = "secret-value-for-rename-test";
+
+        println!("Testing variable rename in Vault...");
+
+        // Create secret at old path
+        println!("  Creating secret at old path: {}", old_path);
+        backend
+            .set_secret(workspace_id, old_path, value)
+            .await
+            .expect("Failed to create secret");
+
+        // Verify it exists
+        let read_value = backend
+            .get_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to read secret at old path");
+        assert_eq!(read_value, value);
+        println!("  ✓ Secret exists at old path");
+
+        // Simulate rename: read from old, write to new, delete old
+        println!("  Renaming: {} -> {}", old_path, new_path);
+        let secret_value = backend
+            .get_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to read for rename");
+
+        backend
+            .set_secret(workspace_id, new_path, &secret_value)
+            .await
+            .expect("Failed to write to new path");
+
+        backend
+            .delete_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to delete old path");
+
+        // Verify old path is gone
+        let old_result = backend.get_secret(workspace_id, old_path).await;
+        assert!(old_result.is_err(), "Old path should not exist");
+        println!("  ✓ Old path deleted");
+
+        // Verify new path exists with correct value
+        let new_value = backend
+            .get_secret(workspace_id, new_path)
+            .await
+            .expect("Failed to read new path");
+        assert_eq!(new_value, value);
+        println!("  ✓ New path exists with correct value");
+
+        // Cleanup
+        backend
+            .delete_secret(workspace_id, new_path)
+            .await
+            .expect("Failed to cleanup");
+
+        println!("\n✓ Variable rename completed successfully");
+    }
+
+    // ==================== Full Round Trip Test ====================
+
+    /// Test full round-trip: DB -> Vault -> DB with verification
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_full_round_trip(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // Get original secrets
+        let original: HashMap<(String, String), String> = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query")
+        .into_iter()
+        .map(|r| ((r.workspace_id, r.path), r.value))
+        .collect();
+
+        println!("Original secrets: {} entries", original.len());
+
+        // Step 1: DB -> Vault
+        println!("\n=== Step 1: Migrate DB -> Vault ===");
+        let to_vault = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration to Vault failed");
+        println!("Migrated {} secrets to Vault", to_vault.migrated_count);
+        assert_eq!(to_vault.failed_count, 0);
+
+        // Step 2: Clear DB
+        println!("\n=== Step 2: Clear database values ===");
+        sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
+            .execute(&db)
+            .await
+            .expect("Failed to clear");
+
+        // Step 3: Vault -> DB
+        println!("\n=== Step 3: Migrate Vault -> DB ===");
+        let to_db = migrate_secrets_to_database(&db, &settings)
+            .await
+            .expect("Migration to database failed");
+        println!("Migrated {} secrets to database", to_db.migrated_count);
+        assert_eq!(to_db.failed_count, 0);
+
+        // Step 4: Verify
+        println!("\n=== Step 4: Verify round-trip integrity ===");
+        let restored: HashMap<(String, String), String> = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query")
+        .into_iter()
+        .map(|r| ((r.workspace_id, r.path), r.value))
+        .collect();
+
+        for ((ws, path), _) in &original {
+            let restored_value = restored
+                .get(&(ws.clone(), path.clone()))
+                .expect(&format!("Secret {}/{} not found after round-trip", ws, path));
+
+            assert_ne!(
+                restored_value, "ROUND_TRIP_CLEARED",
+                "Secret {}/{} was not restored",
+                ws, path
+            );
+            println!("  ✓ {}/{}: restored", ws, path);
+        }
+
+        println!("\n✓ Full round-trip completed successfully!");
+    }
+
+    // ==================== Workspace Isolation Test ====================
+
+    /// Test that workspace isolation is maintained
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_workspace_isolation(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings.clone());
+
+        // First migrate secrets to Vault
+        migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration failed");
+
+        println!("Testing workspace isolation...");
+
+        // Try to access workspace-2 secret from workspace-1 path (should fail)
+        let cross_access = backend
+            .get_secret("test-workspace", "u/test-user/other_secret")
+            .await;
+
+        assert!(
+            cross_access.is_err(),
+            "Cross-workspace access should fail!"
+        );
+        println!("✓ Cross-workspace access correctly denied");
+
+        // Verify own workspace access works
+        let ws1 = backend
+            .get_secret("test-workspace", "u/test-user/db_password")
+            .await;
+        assert!(ws1.is_ok(), "Same-workspace access should work");
+        println!("✓ Same-workspace access works");
+
+        println!("\n✓ Workspace isolation verified!");
+    }
+}
+
+// OSS version - just a placeholder to avoid compilation errors
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+mod tests {
+    #[test]
+    fn test_vault_requires_enterprise() {
+        println!("Vault integration tests require Enterprise Edition features");
+        println!("Run with: cargo test --features private,enterprise,openidconnect");
+    }
+}

backend/tests/secret_backend_migration.rs

@@ -0,0 +1,323 @@
+//! Integration tests for secret backend migration between database and HashiCorp Vault.
+//!
+//! These tests require:
+//! 1. A PostgreSQL database (handled by sqlx test framework)
+//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200
+//!
+//! To run these tests:
+//! ```bash
+//! # Start Vault in dev mode
+//! podman run -d --name vault-test -p 8200:8200 \
+//!     -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \
+//!     docker.io/hashicorp/vault:latest
+//!
+//! # Enable KV v2 secrets engine
+//! curl -s -H "X-Vault-Token: test-root-token" -X POST \
+//!     --data '{"type":"kv-v2"}' \
+//!     http://127.0.0.1:8200/v1/sys/mounts/windmill
+//!
+//! # Run the tests
+//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture
+//! ```
+
+use sqlx::{Pool, Postgres};
+use windmill_common::error::Result;
+use windmill_common::secret_backend::{
+    vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend},
+    SecretBackend, VaultSettings,
+};
+
+mod common;
+
+fn test_vault_settings() -> VaultSettings {
+    VaultSettings {
+        address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+        mount_path: "windmill".to_string(),
+        jwt_role: Some("windmill-secrets".to_string()),
+        namespace: None,
+        token: Some(
+            std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
+        ),
+    }
+}
+
+/// Test that we can connect to Vault
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_vault_connection_works(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    let result = test_vault_connection(&settings, Some(&db)).await;
+    assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err());
+    println!("✓ Successfully connected to Vault at {}", settings.address);
+}
+
+/// Test migration from database to Vault
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // First verify we can connect to Vault
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Check initial state - secrets should exist in database
+    let secrets_before = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query secrets");
+
+    println!("Found {} secrets in database before migration:", secrets_before.len());
+    for s in &secrets_before {
+        println!("  - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+    }
+
+    // Run migration to Vault
+    println!("\nMigrating secrets to Vault...");
+    let report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration to Vault failed");
+
+    println!("Migration report:");
+    println!("  Total secrets: {}", report.total_secrets);
+    println!("  Migrated: {}", report.migrated_count);
+    println!("  Failed: {}", report.failed_count);
+
+    if !report.failures.is_empty() {
+        println!("  Failures:");
+        for f in &report.failures {
+            println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+        }
+    }
+
+    assert_eq!(report.failed_count, 0, "Migration had failures");
+    assert!(report.migrated_count > 0, "No secrets were migrated");
+
+    // Verify secrets are in Vault
+    println!("\nVerifying secrets in Vault...");
+    let vault_backend = VaultBackend::new(settings.clone());
+
+    for secret in &secrets_before {
+        let result: Result<String> = vault_backend
+            .get_secret(&secret.workspace_id, &secret.path)
+            .await;
+        assert!(
+            result.is_ok(),
+            "Failed to read secret {}/{} from Vault: {:?}",
+            secret.workspace_id,
+            secret.path,
+            result.err()
+        );
+        println!("  ✓ {}/{} exists in Vault", secret.workspace_id, secret.path);
+    }
+
+    println!("\n✓ Migration to Vault completed successfully");
+}
+
+/// Test migration from Vault to database
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // First verify we can connect to Vault
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // First, migrate secrets TO Vault so we have something to migrate back
+    println!("Setting up: migrating secrets to Vault first...");
+    let to_vault_report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Initial migration to Vault failed");
+    assert!(to_vault_report.migrated_count > 0, "No secrets to test with");
+    println!("  Migrated {} secrets to Vault", to_vault_report.migrated_count);
+
+    // Clear the database values to simulate fresh migration back
+    println!("\nClearing database secret values...");
+    sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
+        .execute(&db)
+        .await
+        .expect("Failed to clear database values");
+
+    // Verify they were cleared
+    let cleared = sqlx::query!(
+        "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'"
+    )
+    .fetch_one(&db)
+    .await
+    .expect("Failed to count cleared");
+    println!("  Cleared {} secret values in database", cleared.count.unwrap_or(0));
+
+    // Now migrate from Vault back to database
+    println!("\nMigrating secrets from Vault to database...");
+    let report = migrate_secrets_to_database(&db, &settings)
+        .await
+        .expect("Migration to database failed");
+
+    println!("Migration report:");
+    println!("  Total secrets: {}", report.total_secrets);
+    println!("  Migrated: {}", report.migrated_count);
+    println!("  Failed: {}", report.failed_count);
+
+    if !report.failures.is_empty() {
+        println!("  Failures:");
+        for f in &report.failures {
+            println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+        }
+    }
+
+    assert_eq!(report.failed_count, 0, "Migration had failures");
+    assert!(report.migrated_count > 0, "No secrets were migrated");
+
+    // Verify secrets are restored in database
+    println!("\nVerifying secrets in database...");
+    let secrets_after = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query restored secrets");
+
+    assert!(
+        !secrets_after.is_empty(),
+        "No secrets were restored in database"
+    );
+
+    for s in &secrets_after {
+        println!("  ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+    }
+
+    println!("\n✓ Migration to database completed successfully");
+}
+
+/// Test full round-trip migration: DB -> Vault -> DB
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_full_round_trip_migration(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // Verify Vault connection
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Get original secrets
+    let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query original secrets")
+    .into_iter()
+    .map(|r| ((r.workspace_id, r.path), r.value))
+    .collect();
+
+    println!("Original secrets: {} entries", original_secrets.len());
+
+    // Step 1: Migrate to Vault
+    println!("\n=== Step 1: Migrate DB -> Vault ===");
+    let to_vault = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration to Vault failed");
+    println!("Migrated {} secrets to Vault", to_vault.migrated_count);
+    assert_eq!(to_vault.failed_count, 0);
+
+    // Step 2: Clear database values
+    println!("\n=== Step 2: Clear database values ===");
+    sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
+        .execute(&db)
+        .await
+        .expect("Failed to clear values");
+
+    // Step 3: Migrate back from Vault
+    println!("\n=== Step 3: Migrate Vault -> DB ===");
+    let to_db = migrate_secrets_to_database(&db, &settings)
+        .await
+        .expect("Migration to database failed");
+    println!("Migrated {} secrets to database", to_db.migrated_count);
+    assert_eq!(to_db.failed_count, 0);
+
+    // Step 4: Verify round-trip integrity
+    println!("\n=== Step 4: Verify round-trip integrity ===");
+    let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query restored secrets")
+    .into_iter()
+    .map(|r| ((r.workspace_id, r.path), r.value))
+    .collect();
+
+    // Compare original and restored
+    for ((ws, path), _original_value) in &original_secrets {
+        let restored_value = restored_secrets
+            .get(&(ws.clone(), path.clone()))
+            .expect(&format!("Secret {}/{} not found after round-trip", ws, path));
+
+        // Note: Values might differ slightly due to encryption/decryption
+        // but they should not be the cleared value
+        assert_ne!(
+            restored_value, "ROUND_TRIP_CLEARED",
+            "Secret {}/{} was not restored",
+            ws, path
+        );
+        println!("  ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len());
+    }
+
+    println!("\n✓ Full round-trip migration completed successfully!");
+    println!("  Original secrets: {}", original_secrets.len());
+    println!("  Restored secrets: {}", restored_secrets.len());
+}
+
+/// Test that workspace isolation is maintained during migration
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_workspace_isolation(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Migrate all secrets to Vault
+    let report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration failed");
+
+    println!("Migrated {} secrets across workspaces", report.migrated_count);
+
+    // Verify workspace isolation in Vault
+    let vault_backend = VaultBackend::new(settings.clone());
+
+    // Try to access test-workspace-2 secret from test-workspace path (should fail)
+    let cross_workspace_result: Result<String> = vault_backend
+        .get_secret("test-workspace", "u/test-user/other_secret")
+        .await;
+
+    assert!(
+        cross_workspace_result.is_err(),
+        "Cross-workspace access should fail - workspace isolation violated!"
+    );
+    println!("✓ Cross-workspace access correctly denied");
+
+    // Verify each workspace's secrets are accessible from their own workspace
+    let ws1_result: Result<String> = vault_backend
+        .get_secret("test-workspace", "u/test-user/db_password")
+        .await;
+    assert!(ws1_result.is_ok(), "test-workspace secret should be accessible");
+    println!("✓ test-workspace secrets accessible");
+
+    let ws2_result: Result<String> = vault_backend
+        .get_secret("test-workspace-2", "u/test-user/other_secret")
+        .await;
+    assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible");
+    println!("✓ test-workspace-2 secrets accessible");
+
+    println!("\n✓ Workspace isolation verified!");
+}

backend/tests/success_handler.rs

@@ -0,0 +1,241 @@
+use sqlx::{Pool, Postgres};
+
+mod common;
+use common::*;
+
+/// Test that the workspace success handler cache works correctly with 60s TTL
+#[cfg(feature = "deno_core")]
+#[sqlx::test(fixtures("base"))]
+async fn test_success_handler_cache(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    // First, create a success handler script
+    let _server = ApiServer::start(db.clone()).await?;
+
+    // Set up a success handler in workspace_settings
+    sqlx::query!(
+        r#"
+        UPDATE workspace_settings
+        SET success_handler = 'script/f/test/success_handler',
+            success_handler_extra_args = '{"key": "value"}'::json
+        WHERE workspace_id = 'test-workspace'
+        "#
+    )
+    .execute(&db)
+    .await?;
+
+    // Verify the success handler was set
+    let result = sqlx::query_scalar!(
+        r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
+    )
+    .fetch_one(&db)
+    .await?;
+
+    assert_eq!(result, Some("script/f/test/success_handler".to_string()));
+
+    // Verify extra args were set
+    let extra_args = sqlx::query_scalar!(
+        r#"SELECT success_handler_extra_args FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
+    )
+    .fetch_one(&db)
+    .await?;
+
+    assert!(extra_args.is_some());
+
+    Ok(())
+}
+
+/// Test that success handler can be set and removed via database operations
+#[cfg(feature = "deno_core")]
+#[sqlx::test(fixtures("base"))]
+async fn test_success_handler_settings(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let _server = ApiServer::start(db.clone()).await?;
+
+    // Initially success_handler should be NULL
+    let initial = sqlx::query_scalar!(
+        r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
+    )
+    .fetch_one(&db)
+    .await?;
+    assert!(initial.is_none());
+
+    // Set success handler
+    sqlx::query!(
+        r#"
+        UPDATE workspace_settings
+        SET success_handler = 'flow/f/test/success_flow'
+        WHERE workspace_id = 'test-workspace'
+        "#
+    )
+    .execute(&db)
+    .await?;
+
+    let after_set = sqlx::query_scalar!(
+        r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
+    )
+    .fetch_one(&db)
+    .await?;
+    assert_eq!(after_set, Some("flow/f/test/success_flow".to_string()));
+
+    // Remove success handler
+    sqlx::query!(
+        r#"
+        UPDATE workspace_settings
+        SET success_handler = NULL
+        WHERE workspace_id = 'test-workspace'
+        "#
+    )
+    .execute(&db)
+    .await?;
+
+    let after_remove = sqlx::query_scalar!(
+        r#"SELECT success_handler FROM workspace_settings WHERE workspace_id = 'test-workspace'"#
+    )
+    .fetch_one(&db)
+    .await?;
+    assert!(after_remove.is_none());
+
+    Ok(())
+}
+
+/// Test that a successful job completion triggers the success handler
+#[cfg(all(feature = "deno_core", feature = "enterprise", feature = "private"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_success_handler_triggered_on_success(db: Pool<Postgres>) -> anyhow::Result<()> {
+    use serde_json::json;
+    use windmill_common::jobs::JobPayload;
+    use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings};
+    use windmill_common::scripts::{ScriptHash, ScriptLang};
+
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+
+    // Create a simple success handler script
+    // Note: lock must be non-null for script to be considered "deployed"
+    let success_handler_code = r#"
+export async function main(path: string, email: string, job_id: string, is_flow: boolean, workspace_id: string, result: any) {
+    console.log("Success handler called for job:", job_id);
+    return { handled: true, original_path: path };
+}
+"#;
+
+    sqlx::query!(
+        r#"
+        INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
+        VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')
+        "#,
+        success_handler_code
+    )
+    .execute(&db)
+    .await?;
+
+    // Create a simple test script that we'll run (needs to be JobKind::Script, not Preview)
+    // Note: lock must be non-null for script to be considered "deployed"
+    let test_script_code = "export function main() { return 'success'; }";
+    let test_script_hash: i64 = 9876543210;
+
+    sqlx::query!(
+        r#"
+        INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
+        VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')
+        "#,
+        test_script_hash,
+        test_script_code
+    )
+    .execute(&db)
+    .await?;
+
+    // Set up the success handler in workspace_settings
+    sqlx::query!(
+        r#"
+        UPDATE workspace_settings
+        SET success_handler = 'script/f/test/success_handler'
+        WHERE workspace_id = 'test-workspace'
+        "#
+    )
+    .execute(&db)
+    .await?;
+
+    // Create the success_handler group
+    sqlx::query!(
+        r#"
+        INSERT INTO group_ (workspace_id, name, summary, extra_perms)
+        VALUES ('test-workspace', 'success_handler', 'The group the success handler acts on behalf of', '{"u/test-user": true}')
+        ON CONFLICT DO NOTHING
+        "#
+    )
+    .execute(&db)
+    .await?;
+
+    // Run a script using ScriptHash (produces JobKind::Script, not Preview)
+    let completed_job = RunJob::from(JobPayload::ScriptHash {
+        hash: ScriptHash(test_script_hash),
+        path: "f/test/simple_script".to_string(),
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        language: ScriptLang::Deno,
+        priority: None,
+        apply_preprocessor: false,
+        concurrency_settings: ConcurrencySettings::default(),
+        debouncing_settings: DebouncingSettings::default(),
+    })
+    .run_until_complete(&db, false, server.addr.port())
+    .await;
+
+    let result = completed_job.json_result().unwrap();
+    assert_eq!(result, json!("success"));
+
+    let main_job_id = completed_job.id;
+
+    // Wait a short time for the success handler job to be created
+    tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
+
+    // Verify the success handler job was created (query by email since trigger is not set)
+    let success_handler_job = sqlx::query!(
+        r#"
+        SELECT
+            id,
+            runnable_path,
+            permissioned_as_email,
+            parent_job
+        FROM v2_job
+        WHERE workspace_id = 'test-workspace'
+            AND permissioned_as_email = 'success_handler@windmill.dev'
+        ORDER BY created_at DESC
+        LIMIT 1
+        "#
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    assert!(
+        success_handler_job.is_some(),
+        "Success handler job should have been created"
+    );
+
+    let handler_job = success_handler_job.unwrap();
+
+    // Verify the success handler job has correct parameters
+    assert_eq!(
+        handler_job.runnable_path.as_deref(),
+        Some("f/test/success_handler"),
+        "Success handler should run the configured script"
+    );
+    assert_eq!(
+        handler_job.permissioned_as_email.as_str(),
+        "success_handler@windmill.dev",
+        "Success handler should run as success_handler user"
+    );
+    assert_eq!(
+        handler_job.parent_job,
+        Some(main_job_id),
+        "Success handler should have main job as parent"
+    );
+    // Note: root_job may be None when it equals parent_job (optimization in push function)
+
+    Ok(())
+}

backend/windmill-api/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/lib.rs"
 
 [features]
 default = []
-private = ["windmill-audit/private"]
+private = ["windmill-audit/private", "windmill-common/private"]
 enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
 stripe = []
 agent_worker_server = []
@@ -157,6 +157,8 @@ google-cloud-googleapis = { workspace = true , optional = true }
 tonic = { workspace = true, optional = true }
 deno_error = { workspace = true, optional = true }
 deno_core = { workspace = true, optional = true }
+tar.workspace = true
+flate2.workspace = true
 backon = {workspace = true, optional = true}
 
 [build-dependencies]

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.609.0
+  version: 1.613.4
   title: Windmill API
 
   contact:
@@ -1411,6 +1411,97 @@ paths:
                 items:
                   $ref: "#/components/schemas/GlobalSetting"
 
+  /min_keep_alive_version:
+    get:
+      summary: get minimum worker version required to stay alive
+      operationId: getMinKeepAliveVersion
+      tags:
+        - setting
+      responses:
+        "200":
+          description: minimum keep-alive version
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /.well-known/jwks.json:
+    get:
+      summary: get JWKS for Vault JWT authentication
+      operationId: getJwks
+      tags:
+        - setting
+      responses:
+        "200":
+          description: JSON Web Key Set
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JwksResponse"
+
+  /settings/test_secret_backend:
+    post:
+      summary: test secret backend connection (HashiCorp Vault)
+      operationId: testSecretBackend
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings to test
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: connection successful
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /settings/migrate_secrets_to_vault:
+    post:
+      summary: migrate secrets from database to HashiCorp Vault
+      operationId: migrateSecretsToVault
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings for migration target
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: migration report
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SecretMigrationReport"
+
+  /settings/migrate_secrets_to_database:
+    post:
+      summary: migrate secrets from HashiCorp Vault to database
+      operationId: migrateSecretsToDatabase
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings for migration source
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: migration report
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SecretMigrationReport"
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -2276,6 +2367,10 @@ paths:
                     $ref: "#/components/schemas/ScriptArgs"
                   error_handler_muted_on_cancel:
                     type: boolean
+                  success_handler:
+                    type: string
+                  success_handler_extra_args:
+                    $ref: "#/components/schemas/ScriptArgs"
                   large_file_storage:
                     $ref: "#/components/schemas/LargeFileStorage"
                   ducklake:
@@ -2992,6 +3087,34 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/workspaces/edit_success_handler:
+    post:
+      summary: edit success handler
+      operationId: editSuccessHandler
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: WorkspaceSuccessHandler
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                success_handler:
+                  type: string
+                success_handler_extra_args:
+                  $ref: "#/components/schemas/ScriptArgs"
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/workspaces/edit_large_file_storage_config:
     post:
       summary: edit large file storage settings
@@ -3397,23 +3520,6 @@ paths:
                   default_app_path:
                     type: string
 
-  /w/{workspace}/workspaces/get_large_file_storage_config:
-    get:
-      summary: get large file storage config
-      operationId: getLargeFileStorageConfig
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/LargeFileStorage"
-
   /w/{workspace}/workspaces/usage:
     get:
       summary: get usage
@@ -4977,6 +5083,141 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/npm_proxy/metadata/{package}:
+    get:
+      summary: get npm package metadata from private registry
+      operationId: getNpmPackageMetadata
+      tags:
+        - npm_proxy
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: package
+          description: npm package name
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: package metadata
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tags:
+                    type: object
+                    additionalProperties:
+                      type: string
+                  versions:
+                    type: array
+                    items:
+                      type: string
+
+  /w/{workspace}/npm_proxy/resolve/{package}:
+    get:
+      summary: resolve npm package version from private registry
+      operationId: resolveNpmPackageVersion
+      tags:
+        - npm_proxy
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: package
+          description: npm package name
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: tag
+          description: version tag or reference
+          in: query
+          required: false
+          schema:
+            type: string
+      responses:
+        "200":
+          description: resolved version
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  version:
+                    type: string
+                    nullable: true
+
+  /w/{workspace}/npm_proxy/filetree/{package}/{version}:
+    get:
+      summary: get npm package file tree from private registry
+      operationId: getNpmPackageFiletree
+      tags:
+        - npm_proxy
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: package
+          description: npm package name
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: version
+          description: package version
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: package file tree
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  default:
+                    type: string
+                  files:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        name:
+                          type: string
+
+  /w/{workspace}/npm_proxy/file/{package}/{version}/{filepath}:
+    get:
+      summary: get specific file from npm package in private registry
+      operationId: getNpmPackageFile
+      tags:
+        - npm_proxy
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: package
+          description: npm package name
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: version
+          description: package version
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: filepath
+          description: file path within package
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: file content
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/embeddings/query_resource_types:
     get:
       summary: query resource types by similarity
@@ -5676,11 +5917,6 @@ paths:
       tags:
         - worker
       parameters:
-        - name: workspace
-          in: query
-          schema:
-            type: string
-          required: false
         - name: show_workspace_restriction
           in: query
           schema:
@@ -5696,6 +5932,24 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/workers/custom_tags:
+    get:
+      summary: get custom tags available for this workspace
+      operationId: getCustomTagsForWorkspace
+      tags:
+        - worker
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: list of custom tags for workspace
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
   /workers/get_default_tags:
     get:
       summary: get all instance default tags
@@ -9087,6 +9341,25 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/jobs/get_otel_traces/{id}:
+    get:
+      summary: get OpenTelemetry traces for a job
+      operationId: getJobOtelTraces
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: list of OTEL Span objects (compatible with OpenTelemetry Span proto)
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+
   /w/{workspace}/trigger/{trigger_kind}/resume_suspended_trigger_jobs/{trigger_path}:
     post:
       summary: resume all suspended jobs for a specific trigger
@@ -16430,6 +16703,83 @@ components:
     # -- INLINE END --
     # Do not change line above
 
+    VaultSettings:
+      type: object
+      required:
+        - address
+        - mount_path
+      properties:
+        address:
+          type: string
+          description: HashiCorp Vault server address (e.g., https://vault.company.com:8200)
+        mount_path:
+          type: string
+          description: KV v2 secrets engine mount path (e.g., windmill)
+        jwt_role:
+          type: string
+          description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used)
+        namespace:
+          type: string
+          description: Vault Enterprise namespace (optional)
+        token:
+          type: string
+          description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication)
+
+    SecretMigrationFailure:
+      type: object
+      required:
+        - workspace_id
+        - path
+        - error
+      properties:
+        workspace_id:
+          type: string
+          description: Workspace ID where the secret is located
+        path:
+          type: string
+          description: Path of the secret that failed to migrate
+        error:
+          type: string
+          description: Error message
+
+    SecretMigrationReport:
+      type: object
+      required:
+        - total_secrets
+        - migrated_count
+        - failed_count
+        - failures
+      properties:
+        total_secrets:
+          type: integer
+          format: int64
+          description: Total number of secrets found
+        migrated_count:
+          type: integer
+          format: int64
+          description: Number of secrets successfully migrated
+        failed_count:
+          type: integer
+          format: int64
+          description: Number of secrets that failed to migrate
+        failures:
+          type: array
+          items:
+            $ref: "#/components/schemas/SecretMigrationFailure"
+          description: Details of any failures encountered during migration
+
+    JwksResponse:
+      type: object
+      required:
+        - keys
+      properties:
+        keys:
+          type: array
+          items:
+            type: object
+            additionalProperties: true
+          description: Array of JSON Web Keys for JWT verification
+
     FlowConversation:
       type: object
       required:
@@ -17230,6 +17580,7 @@ components:
         - visible_to_owner
         - tag
 
+
     ExportableCompletedJob:
       type: object
       description: Completed job with full data for export/import operations
@@ -20619,6 +20970,10 @@ components:
             type: boolean
         custom_path:
           type: string
+        raw_app:
+          type: boolean
+        bundle_secret:
+          type: string
       required:
         - id
         - workspace_id
@@ -20631,6 +20986,7 @@ components:
         - policy
         - execution_mode
         - extra_perms
+        - raw_app
 
     AppWithLastVersionWDraft:
       allOf:

backend/windmill-api/src/ai.rs

@@ -6,7 +6,7 @@ use http::{HeaderMap, Method};
 use quick_cache::sync::Cache;
 use reqwest::{Client, RequestBuilder};
 use serde::{Deserialize, Serialize};
-use serde_json::value::RawValue;
+use serde_json::{json, value::RawValue};
 use std::collections::HashMap;
 use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel};
@@ -125,6 +125,15 @@ struct AIOAuthResource {
     user: Option<String>,
 }
 
+/// Platform for Anthropic API
+#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
+#[serde(rename_all = "snake_case")]
+enum AnthropicPlatform {
+    #[default]
+    Standard,
+    GoogleVertexAi,
+}
+
 #[derive(Deserialize, Debug)]
 struct AIStandardResource {
     #[serde(alias = "baseUrl")]
@@ -137,6 +146,9 @@ struct AIStandardResource {
     aws_access_key_id: Option<String>,
     #[serde(alias = "awsSecretAccessKey")]
     aws_secret_access_key: Option<String>,
+    /// Platform for Anthropic API (standard or google_vertex_ai)
+    #[serde(default)]
+    platform: AnthropicPlatform,
 }
 
 #[derive(Deserialize, Debug)]
@@ -161,6 +173,7 @@ struct AIRequestConfig {
     pub region: Option<String>,
     pub aws_access_key_id: Option<String>,
     pub aws_secret_access_key: Option<String>,
+    pub platform: AnthropicPlatform,
 }
 
 impl AIRequestConfig {
@@ -179,9 +192,11 @@ impl AIRequestConfig {
             region,
             aws_access_key_id,
             aws_secret_access_key,
+            platform,
         ) = match resource {
             AIResource::Standard(resource) => {
                 let region = resource.region.clone();
+                let platform = resource.platform.clone();
                 let base_url = provider
                     .get_base_url(resource.base_url, resource.region, db)
                     .await?;
@@ -216,6 +231,7 @@ impl AIRequestConfig {
                     region,
                     aws_access_key_id,
                     aws_secret_access_key,
+                    platform,
                 )
             }
             AIResource::OAuth(resource) => {
@@ -227,7 +243,17 @@ impl AIRequestConfig {
                 let token = Self::get_token_using_oauth(resource, db, w_id).await?;
                 let base_url = provider.get_base_url(None, None, db).await?;
 
-                (None, Some(token), None, base_url, user, None, None, None)
+                (
+                    None,
+                    Some(token),
+                    None,
+                    base_url,
+                    user,
+                    None,
+                    None,
+                    None,
+                    AnthropicPlatform::Standard,
+                )
             }
         };
 
@@ -240,6 +266,7 @@ impl AIRequestConfig {
             region,
             aws_access_key_id,
             aws_secret_access_key,
+            platform,
         })
     }
 
@@ -294,8 +321,18 @@ impl AIRequestConfig {
 
         let is_azure = provider.is_azure_openai(base_url);
         let is_anthropic = matches!(provider, AIProvider::Anthropic);
+        let is_anthropic_vertex = is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi;
         let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some();
         let is_bedrock = matches!(provider, AIProvider::AWSBedrock);
+        let is_google_ai = matches!(provider, AIProvider::GoogleAI);
+
+        // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent
+        let base_url = if is_google_ai {
+            format!("{}/openai", base_url)
+        } else {
+            base_url.to_string()
+        };
+        let base_url = base_url.as_str();
 
         // Check if using IAM credentials for Bedrock (instead of bearer token)
         let use_iam_auth =
@@ -317,6 +354,10 @@ impl AIRequestConfig {
             let bedrock_base_url = base_url.replace("bedrock-runtime.", "bedrock.");
             let bedrock_url = format!("{}/{}", bedrock_base_url, path);
             (bedrock_url, body)
+        } else if is_anthropic_vertex && method != Method::GET {
+            let (model, transformed_body) = transform_anthropic_for_vertex(&body)?;
+            let vertex_url = format!("{}/{}:streamRawPredict", base_url, model);
+            (vertex_url, transformed_body)
         } else if is_azure {
             let azure_url = AIProvider::build_azure_openai_url(base_url, path);
             (azure_url, body)
@@ -336,7 +377,12 @@ impl AIRequestConfig {
             .header("content-type", "application/json");
 
         for (header_name, header_value) in headers.iter() {
+            // Forward anthropic-* headers, but skip anthropic-version for Vertex AI
+            // (Vertex AI requires anthropic_version in the request body, not as a header)
             if header_name.to_string().starts_with("anthropic-") {
+                if is_anthropic_vertex && header_name.as_str() == "anthropic-version" {
+                    continue;
+                }
                 request = request.header(header_name, header_value);
             }
         }
@@ -360,13 +406,14 @@ impl AIRequestConfig {
             }
         } else {
             // For non-IAM auth, use bearer token or API key
-            if let Some(api_key) = self.api_key {
+            if let Some(api_key) = self.api_key.clone() {
                 if is_azure {
                     request = request.header("api-key", api_key.clone())
                 } else {
                     request = request.header("authorization", format!("Bearer {}", api_key.clone()))
                 }
-                if is_anthropic {
+                // For standard Anthropic API, also add X-API-Key header
+                if is_anthropic && !is_anthropic_vertex {
                     request = request.header("X-API-Key", api_key);
                 }
             }
@@ -438,6 +485,81 @@ pub struct AIConfig {
     pub max_tokens_per_model: Option<HashMap<String, i32>>,
 }
 
+/// Anthropic API version for Google Vertex AI
+const ANTHROPIC_VERSION_VERTEX: &str = "vertex-2023-10-16";
+
+/// Transforms an Anthropic request for Google Vertex AI:
+/// - Extracts the model from the body (needed for the URL)
+/// - Adds anthropic_version to the body
+fn transform_anthropic_for_vertex(body: &Bytes) -> Result<(String, Bytes)> {
+    let mut json_body: HashMap<String, serde_json::Value> = serde_json::from_slice(body)
+        .map_err(|e| Error::internal_err(format!("Failed to parse Anthropic request: {}", e)))?;
+
+    // Extract and remove model from body
+    let model = json_body
+        .remove("model")
+        .and_then(|v| v.as_str().map(|s| s.to_string()))
+        .ok_or_else(|| Error::BadRequest("Missing 'model' field in Anthropic request".to_string()))?;
+
+    // Add anthropic_version to body (required for Vertex AI)
+    json_body.insert(
+        "anthropic_version".to_string(),
+        serde_json::Value::String(ANTHROPIC_VERSION_VERTEX.to_string()),
+    );
+
+    let transformed_body = serde_json::to_vec(&json_body)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize Vertex request: {}", e)))?;
+
+    Ok((model, Bytes::from(transformed_body)))
+}
+
+// FIM (Fill-in-the-Middle) simulation for providers that don't support native FIM
+#[derive(Deserialize, Debug)]
+struct FimRequest {
+    model: String,
+    prompt: String,         // code before cursor
+    suffix: Option<String>, // code after cursor
+    temperature: Option<f32>,
+    max_tokens: Option<u32>,
+    stop: Option<Vec<String>>,
+}
+
+/// Checks if the AI provider supports native FIM (Fill-in-the-Middle) endpoint
+fn supports_native_fim(provider: &AIProvider) -> bool {
+    matches!(provider, AIProvider::Mistral)
+}
+
+/// Transforms a FIM request to chat/completions format for providers that don't support native FIM.
+fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> {
+    let fim_req: FimRequest = serde_json::from_slice(body)
+        .map_err(|e| Error::internal_err(format!("Failed to parse FIM request: {}", e)))?;
+
+    let suffix = fim_req.suffix.unwrap_or_default();
+
+    let system_prompt = "You are a code completion assistant. Complete the code at the <CURSOR/> position between the given prefix and suffix. Output ONLY the code that goes at the cursor - no explanations, no markdown, no repeating the prefix or suffix.";
+
+    let user_content = format!(
+        "<PREFIX>\n{}\n<CURSOR/>\n<SUFFIX>\n{}",
+        fim_req.prompt, suffix
+    );
+
+    let chat_req = json!({
+        "model": fim_req.model,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_content}
+        ],
+        "temperature": fim_req.temperature.unwrap_or(0.0),
+        "max_tokens": fim_req.max_tokens.unwrap_or(256),
+        "stop": fim_req.stop
+    });
+
+    let chat_body = serde_json::to_vec(&chat_req)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?;
+
+    Ok((Bytes::from(chat_body), "chat/completions".to_string()))
+}
+
 pub fn global_service() -> Router {
     Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy))
 }
@@ -516,10 +638,10 @@ async fn global_proxy(
 async fn proxy(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
-    Path((w_id, ai_path)): Path<(String, String)>,
+    Path((w_id, mut ai_path)): Path<(String, String)>,
     method: Method,
     headers: HeaderMap,
-    body: Bytes,
+    mut body: Bytes,
 ) -> impl IntoResponse {
     let provider = headers
         .get("X-Provider")
@@ -600,6 +722,19 @@ async fn proxy(
         }
     };
 
+    // Check if this is a FIM request to a provider that doesn't support native FIM endpoint
+    // For such providers, transform to use FIM sentinel tokens with the chat/completions endpoint
+    let is_fim_request = ai_path.contains("fim/completions");
+    if is_fim_request && !supports_native_fim(&provider) {
+        tracing::debug!(
+            "Transforming FIM request to chat/completions with FIM tokens for provider {:?}",
+            provider
+        );
+        let (chat_body, chat_path) = transform_fim_to_chat_completions(&body)?;
+        body = chat_body;
+        ai_path = chat_path;
+    }
+
     // Extract model and streaming flag for Bedrock transformation (only for POST requests)
     let (model, is_streaming) =
         if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST {

backend/windmill-api/src/apps.rs

@@ -120,6 +120,7 @@ pub fn unauthed_service() -> Router {
         .route("/download_s3_file/*path", get(download_s3_file_from_app))
         .route("/public_app/:secret", get(get_public_app_by_secret))
         .route("/public_resource/*path", get(get_public_resource))
+        .route("/get_data/v/*id", get(get_raw_app_data))
 }
 pub fn global_service() -> Router {
     Router::new()
@@ -175,6 +176,9 @@ pub struct AppWithLastVersion {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub custom_path: Option<String>,
     pub raw_app: bool,
+    #[sqlx(skip)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bundle_secret: Option<String>,
 }
 
 #[derive(Serialize, FromRow)]
@@ -770,7 +774,7 @@ async fn get_public_app_by_secret(
         "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
         null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
         app_version.created_at, app_version.created_by, app_version.raw_app
-        FROM app, app_version 
+        FROM app, app_version
         LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
         WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
         .bind(&id)
@@ -778,36 +782,37 @@ async fn get_public_app_by_secret(
     .fetch_optional(&db)
     .await?;
 
-    let app = not_found_if_none(app_o, "App", id.to_string())?;
+    let mut app = not_found_if_none(app_o, "App", id.to_string())?;
 
     let policy = serde_json::from_str::<Policy>(app.policy.0.get()).map_err(to_anyhow)?;
 
-    if matches!(policy.execution_mode, ExecutionMode::Anonymous) {
-        return Ok(Json(app));
-    }
-
-    if opt_authed.is_none() {
-        {
+    if !matches!(policy.execution_mode, ExecutionMode::Anonymous) {
+        if opt_authed.is_none() {
             return Err(Error::NotAuthorized(
                 "App visibility does not allow public access and you are not logged in".to_string(),
             ));
+        } else {
+            let authed = opt_authed.unwrap();
+            let mut tx = user_db.begin(&authed).await?;
+            let is_visible = sqlx::query_scalar!(
+                "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)",
+                id,
+                &w_id
+            )
+            .fetch_one(&mut *tx)
+            .await?;
+            tx.commit().await?;
+            if !is_visible.unwrap_or(false) {
+                return Err(Error::NotAuthorized(
+                    "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(),
+                ));
+            }
         }
-    } else {
-        let authed = opt_authed.unwrap();
-        let mut tx = user_db.begin(&authed).await?;
-        let is_visible = sqlx::query_scalar!(
-            "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)",
-            id,
-            &w_id
-        )
-        .fetch_one(&mut *tx)
-        .await?;
-        tx.commit().await?;
-        if !is_visible.unwrap_or(false) {
-            return Err(Error::NotAuthorized(
-                "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(),
-            ));
-        }
+    }
+
+    // Compute bundle_secret for raw apps
+    if app.raw_app {
+        app.bundle_secret = Some(compute_bundle_secret(&db, &w_id, &app.versions).await?);
     }
 
     Ok(Json(app))
@@ -892,6 +897,15 @@ async fn get_secret_id(
 
 const BUNDLE_SECRET_PREFIX: &str = "bundle_";
 
+pub async fn compute_bundle_secret(db: &DB, w_id: &str, versions: &[i64]) -> Result<String> {
+    let version_id = versions
+        .last()
+        .ok_or_else(|| Error::internal_err("App has no versions".to_string()))?;
+    let mc = build_crypt(db, w_id).await?;
+    let hx = hex::encode(mc.encrypt_str_to_bytes(format!("{}{}", BUNDLE_SECRET_PREFIX, version_id)));
+    Ok(hx)
+}
+
 async fn get_latest_version_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -2249,6 +2263,7 @@ async fn upload_s3_file_from_app(
                     );
                     let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths(
                         &db_with_opt_authed,
+                        Some(&on_behalf_authed),
                         &w_id,
                         None,
                         &[(&file_key, S3Permission::WRITE)],
@@ -2280,6 +2295,7 @@ async fn upload_s3_file_from_app(
                 DbWithOptAuthed::from_authed(&on_behalf_authed, db.clone(), None);
             let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths(
                 &db_with_opt_authed,
+                Some(&on_behalf_authed),
                 &w_id,
                 None,
                 &[(&file_key, S3Permission::WRITE)],
@@ -2319,6 +2335,7 @@ async fn upload_s3_file_from_app(
                 let db_with_opt_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), None);
                 let (_, s3_resource) = get_workspace_s3_resource_and_check_paths(
                     &db_with_opt_authed,
+                Some(&authed),
                     &w_id,
                     None,
                     &[(&file_key, S3Permission::WRITE)],
@@ -2359,7 +2376,7 @@ async fn upload_s3_file_from_app(
     ])
     .into();
 
-    upload_file_from_req(s3_client, &file_key, request, options).await?;
+    let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?;
 
     let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims {
         file_key: file_key.clone(),
@@ -2429,6 +2446,7 @@ async fn delete_s3_file_from_app(
         let db_with_opt_authed = DbWithOptAuthed::from_authed(&on_behalf_authed, db.clone(), None);
         let (_, s3_resource) = get_workspace_s3_resource_and_check_paths(
             &db_with_opt_authed,
+                Some(&on_behalf_authed),
             &w_id,
             None,
             &[(&path.to_string(), S3Permission::DELETE)],

backend/windmill-api/src/azure_proxy_oss.rs

@@ -0,0 +1,6 @@
+// OSS stub for Azure proxy functionality
+// The actual implementation is in azure_proxy_ee.rs (Enterprise Edition)
+
+#[cfg(all(feature = "private", feature = "parquet"))]
+#[allow(unused)]
+pub use crate::azure_proxy_ee::*;

backend/windmill-api/src/concurrency_groups.rs

@@ -224,6 +224,7 @@ async fn get_concurrent_intervals(
             concurrency_key: Some(_),
             allow_wildcards: None,
             trigger_kind: _,
+            include_args: _,
         } => true,
         _ => false,
     };

backend/windmill-api/src/flows.rs

@@ -34,9 +34,10 @@ use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::runnable_settings::RunnableSettingsTrait;
 use windmill_common::utils::query_elems_from_hub;
-use windmill_common::worker::{
-    to_raw_value, CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
+use windmill_common::min_version::{
+    MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
 };
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::HUB_BASE_URL;
 use windmill_common::{
     db::UserDB,
@@ -1476,14 +1477,14 @@ async fn archive_flow_by_path(
 /// Validates that flow debouncing configuration is supported by all workers
 /// Returns an error if debouncing is configured but workers are behind required version
 async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> {
-    if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await
+    if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await
         && !nf.parse_flow_value()?.debouncing_settings.is_default()
     {
         tracing::warn!(
             "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature"
         );
         Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() })
-    } else if !*MIN_VERSION_SUPPORTS_DEBOUNCING_V2.read().await
+    } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
         && !nf
             .parse_flow_value()?
             .debouncing_settings

backend/windmill-api/src/job_helpers_oss.rs

@@ -12,7 +12,7 @@ use windmill_common::s3_helpers::StorageResourceType;
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use crate::db::{ApiAuthed, DB};
 #[cfg(all(feature = "parquet", not(feature = "private")))]
-use object_store::{ObjectStore, PutMultipartOpts};
+use object_store::{ObjectStore, PutMultipartOpts, PutResult};
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use std::sync::Arc;
 #[cfg(not(feature = "private"))]
@@ -85,7 +85,7 @@ pub async fn upload_file_from_req(
     _file_key: &str,
     _req: axum::extract::Request,
     _options: PutMultipartOpts,
-) -> error::Result<()> {
+) -> error::Result<PutResult> {
     Err(error::Error::internal_err(
         "Not implemented in Windmill's Open Source repository".to_string(),
     ))
@@ -152,6 +152,7 @@ pub struct DeleteS3FileQuery {
 #[cfg(not(feature = "private"))]
 pub async fn get_workspace_s3_resource_and_check_paths<'c>(
     _db_with_opt_authed: &DbWithOptAuthed<'c, ApiAuthed>,
+    _authed_api: Option<&ApiAuthed>,
     _w_id: &str,
     _storage: Option<String>,
     _paths: &[(&str, windmill_common::s3_helpers::S3Permission)],

backend/windmill-api/src/jobs.rs

@@ -64,7 +64,9 @@ use crate::{
     concurrency_groups::join_concurrency_key,
     db::{ApiAuthed, DB},
     triggers::trigger_helpers::RunnableId,
-    users::{get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed},
+    users::{
+        get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed,
+    },
     utils::{check_scopes, content_plain, require_super_admin},
 };
 use anyhow::Context;
@@ -341,6 +343,7 @@ pub fn workspaced_service() -> Router {
             "/send_email_with_instance_smtp",
             post(send_email_with_instance_smtp),
         )
+        .route("/get_otel_traces/:id", get(get_otel_traces))
 }
 
 pub fn workspace_unauthed_service() -> Router {
@@ -1765,6 +1768,8 @@ pub struct ListableCompletedJob {
     pub priority: Option<i16>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub labels: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub args: Option<serde_json::Value>,
 }
 
 #[derive(Debug, Deserialize, Clone, Default)]
@@ -1852,6 +1857,7 @@ pub struct ListQueueQuery {
     pub allow_wildcards: Option<bool>,
     pub trigger_kind: Option<JobTriggerKind>,
     pub trigger_path: Option<String>,
+    pub include_args: Option<bool>,
 }
 
 impl From<ListCompletedQuery> for ListQueueQuery {
@@ -1885,6 +1891,7 @@ impl From<ListCompletedQuery> for ListQueueQuery {
             allow_wildcards: lcq.allow_wildcards,
             trigger_kind: lcq.trigger_kind,
             trigger_path: lcq.trigger_path,
+            include_args: lcq.include_args,
         }
     }
 }
@@ -2074,6 +2081,16 @@ async fn list_queue_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListQueueQuery>,
 ) -> error::JsonResult<Vec<ListableQueuedJob>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
+    let args_field = if include_args { "v2_job.args" } else { "null as args" };
+
     let sql = list_queue_jobs_query(
         &w_id,
         &lq,
@@ -2086,7 +2103,7 @@ async fn list_queue_jobs(
             "v2_job_queue.scheduled_for",
             "v2_job.runnable_id as script_hash",
             "v2_job.runnable_path as script_path",
-            "null as args",
+            args_field,
             "v2_job.kind as job_kind",
             "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path",
             "v2_job.permissioned_as",
@@ -2413,6 +2430,14 @@ async fn list_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<Job>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
 
@@ -2425,13 +2450,36 @@ async fn list_jobs(
             "cannot specify both success and running".to_string(),
         ));
     }
+
+    // Create dynamic field arrays when include_args is true
+    let cj_fields: Vec<&str>;
+    let qj_fields: Vec<&str>;
+    let cj_fields_ref: &[&str];
+    let qj_fields_ref: &[&str];
+
+    if include_args {
+        cj_fields = UnifiedJob::completed_job_fields()
+            .iter()
+            .map(|f| if *f == "null as args" { "v2_job.args" } else { *f })
+            .collect();
+        qj_fields = UnifiedJob::queued_job_fields()
+            .iter()
+            .map(|f| if *f == "null as args" { "v2_job.args" } else { *f })
+            .collect();
+        cj_fields_ref = &cj_fields;
+        qj_fields_ref = &qj_fields;
+    } else {
+        cj_fields_ref = UnifiedJob::completed_job_fields();
+        qj_fields_ref = UnifiedJob::queued_job_fields();
+    }
+
     let sqlc = if lq.running.is_none() {
         Some(list_completed_jobs_query(
             &w_id,
             Some(per_page),
             0,
             &ListCompletedQuery { order_desc: Some(true), ..lqc },
-            UnifiedJob::completed_job_fields(),
+            cj_fields_ref,
             true,
             get_scope_tags(&authed),
         ))
@@ -2449,7 +2497,7 @@ async fn list_jobs(
         let mut sqlq = list_queue_jobs_query(
             &w_id,
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
-            UnifiedJob::queued_job_fields(),
+            qj_fields_ref,
             Pagination { per_page: None, page: None },
             true,
             get_scope_tags(&authed),
@@ -3164,10 +3212,22 @@ pub async fn get_resume_urls_internal(
             "{base_url}/approve/{w_id}/{target_job_id}/{resume_id}/{signature}{approver_query}"
         ),
         cancel: build_resume_url(
-            "cancel", &w_id, &target_job_id, &resume_id, &signature, &approver_query, &base_url,
+            "cancel",
+            &w_id,
+            &target_job_id,
+            &resume_id,
+            &signature,
+            &approver_query,
+            &base_url,
         ),
         resume: build_resume_url(
-            "resume", &w_id, &target_job_id, &resume_id, &signature, &approver_query, &base_url,
+            "resume",
+            &w_id,
+            &target_job_id,
+            &resume_id,
+            &signature,
+            &approver_query,
+            &base_url,
         ),
     };
 
@@ -3446,6 +3506,7 @@ pub struct UnifiedJob {
     pub running: Option<bool>,
     pub script_hash: Option<ScriptHash>,
     pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
     pub duration_ms: Option<i64>,
     pub success: Option<bool>,
     pub deleted: bool,
@@ -3566,6 +3627,7 @@ impl UnifiedJob {
 
 impl<'a> From<UnifiedJob> for Job {
     fn from(uj: UnifiedJob) -> Self {
+        let args = uj.args.and_then(|v| serde_json::from_value(v).ok());
         match uj.typ.as_ref() {
             "CompletedJob" => Job::CompletedJob(JobExtended::new(
                 uj.self_wait_time_ms,
@@ -3582,7 +3644,7 @@ impl<'a> From<UnifiedJob> for Job {
                     success: uj.success.unwrap(),
                     script_hash: uj.script_hash,
                     script_path: uj.script_path,
-                    args: None,
+                    args: args.clone(),
                     result: None,
                     result_columns: None,
                     logs: None,
@@ -3622,7 +3684,7 @@ impl<'a> From<UnifiedJob> for Job {
                     script_hash: uj.script_hash,
                     script_path: uj.script_path,
                     script_entrypoint_override: None,
-                    args: None,
+                    args,
                     logs: None,
                     canceled: uj.canceled,
                     canceled_by: uj.canceled_by,
@@ -5108,7 +5170,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
                     if success {
                         StatusCode::OK
                     } else {
-                        StatusCode::INTERNAL_SERVER_ERROR
+                        StatusCode::UNPROCESSABLE_ENTITY
                     },
                     Json(result),
                 )
@@ -5122,7 +5184,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
                 })
                 .unwrap_or_else(|| {
                     if !success {
-                        Ok(StatusCode::INTERNAL_SERVER_ERROR)
+                        Ok(StatusCode::UNPROCESSABLE_ENTITY)
                     } else if result_value.is_some() {
                         Ok(StatusCode::OK)
                     } else {
@@ -5172,7 +5234,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
             if success {
                 StatusCode::OK
             } else {
-                StatusCode::INTERNAL_SERVER_ERROR
+                StatusCode::UNPROCESSABLE_ENTITY
             },
             Json(result),
         )
@@ -8331,6 +8393,7 @@ pub struct ListCompletedQuery {
     pub allow_wildcards: Option<bool>,
     pub trigger_kind: Option<JobTriggerKind>,
     pub trigger_path: Option<String>,
+    pub include_args: Option<bool>,
 }
 
 async fn list_completed_jobs(
@@ -8340,8 +8403,18 @@ async fn list_completed_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<ListableCompletedJob>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
     let (per_page, offset) = paginate(pagination);
 
+    let args_field = if include_args { "v2_job.args" } else { "null as args" };
+
     let sql = list_completed_jobs_query(
         &w_id,
         Some(per_page),
@@ -8377,6 +8450,7 @@ async fn list_completed_jobs(
             "v2_job.tag",
             "v2_job.priority",
             "v2_job_completed.result->'wm_labels' as labels",
+            args_field,
             "'CompletedJob' as type",
         ],
         false,
@@ -8776,3 +8850,63 @@ async fn delete_completed_job<'a>(
     )
     .await;
 }
+
+async fn get_otel_traces(
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> error::Result<Json<Vec<serde_json::Value>>> {
+    // Check job exists and user has permission to view it
+    let job = sqlx::query_scalar!(
+        "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
+        id,
+        w_id
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    match job {
+        Some(created_by) => {
+            if opt_authed.is_none() && created_by != "anonymous" {
+                return Err(Error::BadRequest(
+                    "As a non logged in user, you can only see jobs ran by anonymous users"
+                        .to_string(),
+                ));
+            }
+        }
+        None => {
+            return Err(Error::NotFound(format!("Job {} not found", id)));
+        }
+    }
+
+    let trace_id = id.as_bytes().as_slice();
+
+    let traces = sqlx::query_scalar!(
+        r#"SELECT json_build_object(
+            'trace_id', encode(trace_id, 'hex'),           -- BYTEA to hex string
+            'span_id', encode(span_id, 'hex'),             -- BYTEA to hex string
+            'parent_span_id', encode(parent_span_id, 'hex'), -- BYTEA to hex string
+            'trace_state', trace_state,
+            'flags', flags,
+            'name', name,
+            'kind', kind,
+            'start_time_unix_nano', start_time_unix_nano,
+            'end_time_unix_nano', end_time_unix_nano,
+            'attributes', attributes,
+            'dropped_attributes_count', dropped_attributes_count,
+            'events', events,
+            'dropped_events_count', dropped_events_count,
+            'links', links,
+            'dropped_links_count', dropped_links_count,
+            'status', status
+        ) as "span!"
+        FROM otel_traces
+        WHERE trace_id = $1
+        ORDER BY start_time_unix_nano"#,
+        trace_id
+    )
+    .fetch_all(&db)
+    .await?;
+
+    Ok(Json(traces))
+}

backend/windmill-api/src/lib.rs

@@ -78,6 +78,9 @@ pub mod args;
 mod assets;
 mod audit;
 pub mod auth;
+#[cfg(all(feature = "private", feature = "parquet"))]
+pub mod azure_proxy_ee;
+mod azure_proxy_oss;
 mod bedrock;
 mod capture;
 mod concurrency_groups;
@@ -106,6 +109,7 @@ mod inkeep_oss;
 mod inputs;
 mod integration;
 mod live_migrations;
+mod npm_proxy;
 #[cfg(feature = "http_trigger")]
 mod openapi;
 #[cfg(all(feature = "private", feature = "parquet"))]
@@ -147,6 +151,7 @@ pub mod scim_ee;
 mod scim_oss;
 mod scopes;
 mod scripts;
+mod secret_backend_ext;
 mod service_logs;
 mod settings;
 mod slack_approvals;
@@ -409,13 +414,15 @@ pub async fn run_server(
     let (mcp_router, mcp_cancellation_token) = {
         #[cfg(feature = "mcp")]
         if server_mode || mcp_mode {
+            use mcp::add_www_authenticate_header;
             let (mcp_router, mcp_cancellation_token) =
                 setup_mcp_server(db.clone(), user_db).await?;
-            let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id);
-            (
-                mcp_router.layer(mcp_middleware),
-                Some(mcp_cancellation_token),
-            )
+            // Apply middleware: auth check inside WWW-Authenticate wrapper so 401s get the header
+            let mcp_router = mcp_router
+                .route_layer(from_extractor::<ApiAuthed>())
+                .layer(axum::middleware::from_fn(add_www_authenticate_header))
+                .layer(axum::middleware::from_fn(extract_and_store_workspace_id));
+            (mcp_router, Some(mcp_cancellation_token))
         } else {
             (Router::new(), None)
         }
@@ -495,7 +502,18 @@ pub async fn run_server(
                             #[cfg(not(feature = "oauth2"))]
                             Router::new()
                         })
+                        .nest("/mcp/oauth/server", {
+                            #[cfg(feature = "mcp")]
+                            {
+                                // Only /approve requires authentication (called by frontend)
+                                mcp::oauth_server::workspaced_authed_service()
+                            }
+
+                            #[cfg(not(feature = "mcp"))]
+                            Router::new()
+                        })
                         .nest("/ai", ai::workspaced_service())
+                        .nest("/npm_proxy", npm_proxy::workspaced_service())
                         .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/resources", resources::workspaced_service())
                         .nest("/schedules", schedule::workspaced_service())
@@ -505,6 +523,7 @@ pub async fn run_server(
                             users::workspaced_service().layer(Extension(argon2.clone())),
                         )
                         .nest("/variables", variables::workspaced_service())
+                        .nest("/workers", workers::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
                         .nest("/oidc", oidc_oss::workspaced_service())
                         .nest("/openapi", {
@@ -538,10 +557,20 @@ pub async fn run_server(
                 .nest("/embeddings", embeddings::global_service())
                 .nest("/ai", ai::global_service())
                 .nest("/inkeep", inkeep_oss::global_service())
-                .nest("/mcp/w/:workspace_id/sse", mcp_router)
                 .nest("/mcp/w/:workspace_id/list_tools", mcp_list_tools_service)
                 .route_layer(from_extractor::<ApiAuthed>())
                 .route_layer(from_extractor::<users::Tokened>())
+                // Workspace-scoped OAuth endpoints that don't require authentication
+                // (authorize and token are called by MCP client before user is authenticated)
+                .nest("/w/:workspace_id/mcp/oauth/server", {
+                    #[cfg(feature = "mcp")]
+                    {
+                        mcp::oauth_server::workspaced_unauthed_service()
+                    }
+
+                    #[cfg(not(feature = "mcp"))]
+                    Router::new()
+                })
                 .nest("/jobs", jobs::global_root_service())
                 .nest(
                     "/srch/w/:workspace_id/index",
@@ -580,6 +609,9 @@ pub async fn run_server(
                         .layer(cors.clone()),
                 )
                 .layer(from_extractor::<OptAuthed>())
+                // Deprecated, here for backwards compatibility: user should use /mcp/w/:workspace_id/mcp instead
+                .nest("/mcp/w/:workspace_id/sse", mcp_router.clone())
+                .nest("/mcp/w/:workspace_id/mcp", mcp_router)
                 .nest("/agent_workers", {
                     #[cfg(feature = "agent_worker_server")]
                     {
@@ -682,6 +714,15 @@ pub async fn run_server(
                     #[cfg(not(feature = "mcp"))]
                     Router::new()
                 })
+                .nest("/mcp/oauth/server", {
+                    #[cfg(feature = "mcp")]
+                    {
+                        mcp::oauth_server::global_service().layer(cors.clone())
+                    }
+
+                    #[cfg(not(feature = "mcp"))]
+                    Router::new()
+                })
                 .nest("/r", {
                     #[cfg(feature = "http_trigger")]
                     {
@@ -712,11 +753,44 @@ pub async fn run_server(
                     }
                 })
                 .route("/version", get(git_v))
+                .route("/min_keep_alive_version", get(min_keep_alive_version))
                 .route("/uptodate", get(is_up_to_date))
                 .route("/ee_license", get(ee_license))
                 .route("/openapi.yaml", get(openapi))
                 .route("/openapi.json", get(openapi_json)),
         )
+        // Clients must use workspace-scoped OAuth metadata at:
+        // /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server
+        // This is discovered via /.well-known/oauth-protected-resource?workspace_id=...
+        .route(
+            "/.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server",
+            {
+                #[cfg(feature = "mcp")]
+                {
+                    get(mcp::oauth_server::workspaced_oauth_metadata)
+                }
+                #[cfg(not(feature = "mcp"))]
+                {
+                    get(|| async { axum::http::StatusCode::NOT_FOUND })
+                }
+            },
+        )
+        // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp
+        .route(
+            "/.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp",
+            {
+                #[cfg(feature = "mcp")]
+                {
+                    get(mcp::oauth_server::protected_resource_metadata_by_path)
+                }
+                #[cfg(not(feature = "mcp"))]
+                {
+                    get(|| async { axum::http::StatusCode::NOT_FOUND })
+                }
+            },
+        )
+        // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix)
+        .route("/.well-known/jwks.json", get(settings::get_jwks))
         .fallback(static_assets::static_handler)
         .layer(middleware_stack);
 
@@ -831,6 +905,11 @@ async fn git_v() -> String {
     format!("CE {GIT_VERSION}")
 }
 
+async fn min_keep_alive_version() -> String {
+    let v = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
+    format!("{}.{}.{}", v.0, v.1, v.2)
+}
+
 #[cfg(not(feature = "enterprise"))]
 async fn ee_license() -> &'static str {
     ""

backend/windmill-api/src/live_migrations.rs

@@ -10,7 +10,7 @@ use sqlx::Postgres;
 use std::time::Duration;
 use tokio::task::JoinHandle;
 use windmill_common::error::Error;
-use windmill_common::worker::MIN_VERSION_IS_AT_LEAST_1_461;
+use windmill_common::min_version::MIN_VERSION_IS_AT_LEAST_1_461;
 
 use crate::db::{CustomMigrator, DB};
 use sqlx::migrate::Migrate;
@@ -36,7 +36,7 @@ pub async fn custom_migrations(
         let db2 = db.clone();
         let v2jh = tokio::task::spawn(async move {
             loop {
-                if !*MIN_VERSION_IS_AT_LEAST_1_461.read().await {
+                if !MIN_VERSION_IS_AT_LEAST_1_461.met().await {
                     tracing::info!("Waiting for all workers to be at least version 1.461 before applying v2 finalize migration, sleeping for 5s...");
                     tokio::time::sleep(Duration::from_secs(5)).await;
                     continue;

backend/windmill-api/src/mcp/core.rs

@@ -411,6 +411,53 @@ pub async fn extract_and_store_workspace_id(
     next.run(request).await
 }
 
+/// Middleware that adds WWW-Authenticate header to 401 responses
+/// This helps MCP clients discover the OAuth authorization server (RFC 9728)
+pub async fn add_www_authenticate_header(
+    request: Request<axum::body::Body>,
+    next: Next,
+) -> Response {
+    use axum::http::StatusCode;
+    use windmill_common::BASE_URL;
+
+    // Extract workspace_id before consuming the request
+    let Some(workspace_id) = request
+        .extensions()
+        .get::<WorkspaceId>()
+        .map(|w| w.0.clone())
+    else {
+        return Response::builder()
+            .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
+            .body(axum::body::Body::from("Missing workspace_id in request"))
+            .unwrap();
+    };
+
+    let response = next.run(request).await;
+
+    // Only add header to 401 Unauthorized responses
+    if response.status() == StatusCode::UNAUTHORIZED {
+        let base_url = BASE_URL.read().await;
+
+        // RFC 9728: The resource parameter contains the protected resource URL.
+        // Clients derive the metadata URL by inserting /.well-known/oauth-protected-resource
+        // after the host, e.g., http://host/.well-known/oauth-protected-resource/api/mcp/w/test/mcp
+        let resource_url = format!("{}/api/mcp/w/{}/mcp", base_url, workspace_id);
+        let www_authenticate = format!("Bearer resource=\"{}\"", resource_url);
+
+        // Reconstruct response with the new header
+        let (mut parts, body) = response.into_parts();
+        parts.headers.insert(
+            axum::http::header::WWW_AUTHENTICATE,
+            www_authenticate
+                .parse()
+                .unwrap_or_else(|_| "Bearer".parse().unwrap()),
+        );
+        Response::from_parts(parts, body)
+    } else {
+        response
+    }
+}
+
 /// Setup the MCP server with HTTP transport
 pub async fn setup_mcp_server(
     db: DB,

backend/windmill-api/src/mcp/mod.rs

@@ -8,4 +8,8 @@ mod core;
 mod utils;
 
 // Re-export only what's needed externally
-pub use core::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server};
+pub mod oauth_server;
+pub use core::{
+    add_www_authenticate_header, extract_and_store_workspace_id, list_tools_service,
+    setup_mcp_server,
+};

backend/windmill-api/src/mcp/oauth_server.rs

@@ -0,0 +1,840 @@
+//! OAuth 2.0 Authorization Server for MCP (RFC 6749, 7591, 7636, 8414, 9728)
+
+use axum::{
+    extract::{Extension, Path, Query},
+    response::{IntoResponse, Redirect},
+    routing::{get, post},
+    Form, Json, Router,
+};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use sqlx::FromRow;
+use windmill_common::{
+    error::{Error, Result},
+    utils::rd_string,
+    BASE_URL, DB,
+};
+
+use crate::db::ApiAuthed;
+
+/// Token expiration for MCP OAuth tokens (1 week in seconds)
+const MCP_OAUTH_TOKEN_EXPIRATION_SECS: u64 = 7 * 24 * 60 * 60;
+
+/// Refresh token expiration for MCP OAuth (30 days in seconds)
+const MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS: u64 = 30 * 24 * 60 * 60;
+
+/// RFC 8414
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AuthorizationMetadata {
+    pub issuer: String,
+    pub authorization_endpoint: String,
+    pub token_endpoint: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub registration_endpoint: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scopes_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub response_types_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub grant_types_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub code_challenge_methods_supported: Option<Vec<String>>,
+}
+
+/// RFC 9728
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ProtectedResourceMetadata {
+    pub resource: String,
+    pub authorization_servers: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scopes_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bearer_methods_supported: Option<Vec<String>>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct OAuthJsonError {
+    pub error: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_description: Option<String>,
+}
+
+impl OAuthJsonError {
+    fn new(error: &str, description: Option<&str>) -> Self {
+        Self { error: error.to_string(), error_description: description.map(|s| s.to_string()) }
+    }
+}
+
+impl IntoResponse for OAuthJsonError {
+    fn into_response(self) -> axum::response::Response {
+        (axum::http::StatusCode::BAD_REQUEST, Json(self)).into_response()
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct OAuthTokenError {
+    pub error: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_description: Option<String>,
+}
+
+impl OAuthTokenError {
+    fn invalid_request(description: &str) -> Self {
+        Self {
+            error: "invalid_request".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn invalid_grant(description: &str) -> Self {
+        Self {
+            error: "invalid_grant".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn unsupported_grant_type(description: &str) -> Self {
+        Self {
+            error: "unsupported_grant_type".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn server_error(description: &str) -> Self {
+        Self { error: "server_error".to_string(), error_description: Some(description.to_string()) }
+    }
+}
+
+impl IntoResponse for OAuthTokenError {
+    fn into_response(self) -> axum::response::Response {
+        let status = match self.error.as_str() {
+            "server_error" => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+            _ => axum::http::StatusCode::BAD_REQUEST,
+        };
+        (status, Json(self)).into_response()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ClientRegistrationRequest {
+    pub client_name: String,
+    pub redirect_uris: Vec<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ClientRegistrationResponse {
+    pub client_id: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub client_secret: Option<String>,
+    pub client_name: String,
+    pub redirect_uris: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AuthorizeQuery {
+    pub response_type: String,
+    pub client_id: String,
+    pub redirect_uri: String,
+    #[serde(default)]
+    pub scope: Option<String>,
+    #[serde(default)]
+    pub state: Option<String>,
+    #[serde(default)]
+    pub code_challenge: Option<String>,
+    #[serde(default)]
+    pub code_challenge_method: Option<String>,
+    #[serde(default)]
+    pub resource: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ApprovalForm {
+    pub client_id: String,
+    pub redirect_uri: String,
+    pub scope: String,
+    pub state: String,
+    pub code_challenge: String,
+    pub code_challenge_method: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TokenRequest {
+    pub grant_type: String,
+    #[serde(default)]
+    pub code: String,
+    #[serde(default)]
+    pub redirect_uri: String,
+    #[serde(default)]
+    pub client_id: String,
+    #[serde(default)]
+    pub code_verifier: Option<String>,
+    #[serde(default)]
+    pub refresh_token: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TokenResponse {
+    pub access_token: String,
+    pub token_type: String,
+    pub expires_in: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scope: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub refresh_token: Option<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct OAuthClient {
+    client_id: String,
+    client_name: String,
+    redirect_uris: Vec<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct AuthorizationCode {
+    code: String,
+    client_id: String,
+    user_email: String,
+    workspace_id: String,
+    scopes: Vec<String>,
+    redirect_uri: String,
+    code_challenge: Option<String>,
+    code_challenge_method: Option<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct RefreshTokenRow {
+    id: i64,
+    refresh_token: String,
+    access_token: String,
+    client_id: String,
+    user_email: String,
+    workspace_id: String,
+    scopes: Vec<String>,
+    token_family: sqlx::types::Uuid,
+    created_at: chrono::DateTime<chrono::Utc>,
+    expires_at: chrono::DateTime<chrono::Utc>,
+    used_at: Option<chrono::DateTime<chrono::Utc>>,
+    revoked: bool,
+}
+
+fn supported_scopes() -> Vec<String> {
+    vec![
+        "mcp:all".to_string(),
+        "mcp:favorites".to_string(),
+        "mcp:scripts:*".to_string(),
+        "mcp:flows:*".to_string(),
+        "mcp:endpoints:*".to_string(),
+    ]
+}
+
+/// GET /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server
+pub async fn workspaced_oauth_metadata(
+    Path(workspace_id): Path<String>,
+) -> Json<AuthorizationMetadata> {
+    let base_url = BASE_URL.read().await;
+    let issuer = format!("{}/api/w/{}/mcp/oauth/server", base_url, workspace_id);
+
+    Json(AuthorizationMetadata {
+        issuer,
+        authorization_endpoint: format!(
+            "{}/api/w/{}/mcp/oauth/server/authorize",
+            base_url, workspace_id
+        ),
+        token_endpoint: format!("{}/api/w/{}/mcp/oauth/server/token", base_url, workspace_id),
+        registration_endpoint: Some(format!("{}/api/mcp/oauth/server/register", base_url)),
+        scopes_supported: Some(supported_scopes()),
+        response_types_supported: Some(vec!["code".to_string()]),
+        grant_types_supported: Some(vec![
+            "authorization_code".to_string(),
+            "refresh_token".to_string(),
+        ]),
+        code_challenge_methods_supported: Some(vec!["S256".to_string()]),
+    })
+}
+
+/// GET /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp
+pub async fn protected_resource_metadata_by_path(
+    Path(workspace_id): Path<String>,
+) -> Json<ProtectedResourceMetadata> {
+    let base_url = BASE_URL.read().await;
+    let resource_url = format!("{}/api/mcp/w/{}/mcp", base_url, workspace_id);
+    let auth_server_url = format!("{}/api/w/{}/mcp/oauth/server", base_url, workspace_id);
+    Json(ProtectedResourceMetadata {
+        resource: resource_url,
+        authorization_servers: vec![auth_server_url],
+        scopes_supported: Some(supported_scopes()),
+        bearer_methods_supported: Some(vec!["header".to_string()]),
+    })
+}
+
+/// POST /api/mcp/oauth/server/register - dynamic client registration
+pub async fn oauth_register(
+    Extension(db): Extension<DB>,
+    Json(req): Json<ClientRegistrationRequest>,
+) -> Result<(axum::http::StatusCode, Json<ClientRegistrationResponse>)> {
+    if req.redirect_uris.is_empty() {
+        return Err(Error::BadRequest(
+            "At least one redirect_uri is required".to_string(),
+        ));
+    }
+
+    let client_id = format!("mcp-client-{}", rd_string(16));
+
+    sqlx::query!(
+        "INSERT INTO mcp_oauth_server_client (client_id, client_name, redirect_uris)
+         VALUES ($1, $2, $3)",
+        client_id,
+        req.client_name,
+        &req.redirect_uris,
+    )
+    .execute(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Failed to register client: {}", e)))?;
+
+    Ok((
+        axum::http::StatusCode::CREATED,
+        Json(ClientRegistrationResponse {
+            client_id,
+            client_secret: None,
+            client_name: req.client_name,
+            redirect_uris: req.redirect_uris,
+        }),
+    ))
+}
+
+#[derive(Debug, Serialize)]
+pub struct ApprovalResponse {
+    pub code: String,
+    pub state: Option<String>,
+}
+
+/// POST /api/w/:workspace_id/mcp/oauth/server/token - exchange code for token or refresh
+pub async fn oauth_token(
+    Extension(db): Extension<DB>,
+    Form(req): Form<TokenRequest>,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    match req.grant_type.as_str() {
+        "authorization_code" => handle_authorization_code_grant(&db, &req).await,
+        "refresh_token" => handle_refresh_token_grant(&db, &req).await,
+        _ => Err(OAuthTokenError::unsupported_grant_type(
+            "Supported grant types: authorization_code, refresh_token",
+        )),
+    }
+}
+
+/// Handle authorization_code grant type
+async fn handle_authorization_code_grant(
+    db: &DB,
+    req: &TokenRequest,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    let auth_code = match sqlx::query_as!(
+        AuthorizationCode,
+        "DELETE FROM mcp_oauth_server_code
+         WHERE code = $1 AND expires_at > now()
+         RETURNING code, client_id, user_email, workspace_id, scopes, redirect_uri,
+                   code_challenge, code_challenge_method",
+        req.code
+    )
+    .fetch_optional(db)
+    .await
+    {
+        Ok(Some(code)) => code,
+        Ok(None) => {
+            return Err(OAuthTokenError::invalid_grant(
+                "Invalid or expired authorization code",
+            ));
+        }
+        Err(e) => {
+            tracing::error!("Database error consuming auth code: {}", e);
+            return Err(OAuthTokenError::server_error("Database error"));
+        }
+    };
+
+    if auth_code.client_id != req.client_id {
+        return Err(OAuthTokenError::invalid_grant("client_id mismatch"));
+    }
+
+    if auth_code.redirect_uri != req.redirect_uri {
+        return Err(OAuthTokenError::invalid_grant("redirect_uri mismatch"));
+    }
+
+    let challenge = auth_code.code_challenge.as_ref().ok_or_else(|| {
+        OAuthTokenError::invalid_grant("Authorization code missing PKCE challenge")
+    })?;
+
+    let verifier = req
+        .code_verifier
+        .as_ref()
+        .ok_or_else(|| OAuthTokenError::invalid_request("code_verifier is required"))?;
+
+    let method = auth_code.code_challenge_method.as_deref().unwrap_or("S256");
+    if method != "S256" {
+        return Err(OAuthTokenError::invalid_grant(
+            "Only S256 PKCE method is supported",
+        ));
+    }
+
+    if !validate_pkce_s256(verifier, challenge) {
+        return Err(OAuthTokenError::invalid_grant("Invalid code_verifier"));
+    }
+
+    let access_token = rd_string(32);
+    let refresh_token = rd_string(32);
+    let token_family = sqlx::types::Uuid::new_v4();
+    let scopes = auth_code.scopes;
+
+    // Create access token
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)
+         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+        access_token,
+        auth_code.user_email,
+        format!("mcp-oauth-{}", auth_code.client_id),
+        MCP_OAUTH_TOKEN_EXPIRATION_SECS.to_string(),
+        &scopes,
+        auth_code.workspace_id,
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create access token: {}", e);
+        return Err(OAuthTokenError::server_error(
+            "Failed to create access token",
+        ));
+    }
+
+    // Create refresh token
+    let refresh_token_result = sqlx::query!(
+        "INSERT INTO mcp_oauth_refresh_token
+         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+        refresh_token,
+        access_token,
+        auth_code.client_id,
+        auth_code.user_email,
+        auth_code.workspace_id,
+        &scopes,
+        token_family,
+        MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
+    )
+    .execute(db)
+    .await;
+
+    let refresh_token_value = match refresh_token_result {
+        Ok(_) => Some(refresh_token),
+        Err(e) => {
+            tracing::error!("Failed to create refresh token: {}", e);
+            None // Don't include invalid refresh token
+        }
+    };
+
+    Ok(Json(TokenResponse {
+        access_token,
+        token_type: "Bearer".to_string(),
+        expires_in: MCP_OAUTH_TOKEN_EXPIRATION_SECS,
+        scope: Some(scopes.join(" ")),
+        refresh_token: refresh_token_value,
+    }))
+}
+
+/// Handle refresh_token grant type with token rotation and theft detection
+async fn handle_refresh_token_grant(
+    db: &DB,
+    req: &TokenRequest,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    let refresh_token_value = req
+        .refresh_token
+        .as_ref()
+        .ok_or_else(|| OAuthTokenError::invalid_request("refresh_token is required"))?;
+
+    if req.client_id.is_empty() {
+        return Err(OAuthTokenError::invalid_request("client_id is required"));
+    }
+
+    // Atomically claim the refresh token by setting used_at in a single UPDATE.
+    let token_row = match sqlx::query_as!(
+        RefreshTokenRow,
+        "UPDATE mcp_oauth_refresh_token
+         SET used_at = now()
+         WHERE refresh_token = $1
+           AND client_id = $2
+           AND used_at IS NULL
+           AND NOT revoked
+           AND expires_at > now()
+         RETURNING id, refresh_token, access_token, client_id, user_email, workspace_id,
+                   scopes, token_family, created_at, expires_at, used_at, revoked",
+        refresh_token_value,
+        req.client_id
+    )
+    .fetch_optional(db)
+    .await
+    {
+        Ok(Some(row)) => row,
+        Ok(None) => {
+            // Check for token reuse (theft detection) and revoke family if detected
+            if let Ok(Some(family)) = sqlx::query_scalar!(
+                "SELECT token_family FROM mcp_oauth_refresh_token
+                 WHERE refresh_token = $1 AND used_at IS NOT NULL",
+                refresh_token_value
+            )
+            .fetch_optional(db)
+            .await
+            {
+                tracing::warn!("Refresh token reuse detected, revoking family {:?}", family);
+                let _ = sqlx::query!(
+                    "UPDATE mcp_oauth_refresh_token SET revoked = TRUE WHERE token_family = $1",
+                    family
+                )
+                .execute(db)
+                .await;
+            }
+            return Err(OAuthTokenError::invalid_grant("Invalid refresh token"));
+        }
+        Err(e) => {
+            tracing::error!("Database error claiming refresh token: {}", e);
+            return Err(OAuthTokenError::server_error("Database error"));
+        }
+    };
+
+    // Delete old access token
+    if let Err(e) = sqlx::query!("DELETE FROM token WHERE token = $1", token_row.access_token)
+        .execute(db)
+        .await
+    {
+        tracing::error!("Failed to delete old access token: {}", e);
+        // Non-fatal, continue with token creation
+    }
+
+    // Generate new tokens
+    let new_access_token = rd_string(32);
+    let new_refresh_token = rd_string(32);
+    let scopes = token_row.scopes;
+
+    // Create new access token
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)
+         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+        new_access_token,
+        token_row.user_email,
+        format!("mcp-oauth-{}", token_row.client_id),
+        MCP_OAUTH_TOKEN_EXPIRATION_SECS.to_string(),
+        &scopes,
+        token_row.workspace_id,
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create new access token: {}", e);
+        return Err(OAuthTokenError::server_error(
+            "Failed to create access token",
+        ));
+    }
+
+    // Create new refresh token (same token family for tracking)
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO mcp_oauth_refresh_token
+         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+        new_refresh_token,
+        new_access_token,
+        token_row.client_id,
+        token_row.user_email,
+        token_row.workspace_id,
+        &scopes,
+        token_row.token_family,
+        MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create new refresh token: {}", e);
+        // Access token was created, return success without refresh token
+    }
+
+    Ok(Json(TokenResponse {
+        access_token: new_access_token,
+        token_type: "Bearer".to_string(),
+        expires_in: MCP_OAUTH_TOKEN_EXPIRATION_SECS,
+        scope: Some(scopes.join(" ")),
+        refresh_token: Some(new_refresh_token),
+    }))
+}
+
+/// GET /api/w/:workspace_id/mcp/oauth/server/authorize - redirects to consent page
+pub async fn workspaced_oauth_authorize(
+    Extension(db): Extension<DB>,
+    Path(workspace_id): Path<String>,
+    Query(params): Query<AuthorizeQuery>,
+) -> impl IntoResponse {
+    let client = match sqlx::query_as!(
+        OAuthClient,
+        "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+        params.client_id
+    )
+    .fetch_optional(&db)
+    .await
+    {
+        Ok(Some(client)) => client,
+        Ok(None) => {
+            return OAuthJsonError::new("invalid_client", Some("Unknown client_id"))
+                .into_response();
+        }
+        Err(e) => {
+            tracing::error!("Database error looking up client: {}", e);
+            return OAuthJsonError::new("server_error", Some("Database error"))
+                .into_response();
+        }
+    };
+
+    if !client.redirect_uris.contains(&params.redirect_uri) {
+        return OAuthJsonError::new(
+            "invalid_request",
+            Some("redirect_uri does not match registered URIs"),
+        )
+        .into_response();
+    }
+
+    if params.response_type != "code" {
+        return OAuthErrorRedirect::new(
+            &params.redirect_uri,
+            "unsupported_response_type",
+            Some("Only 'code' response type is supported"),
+            params.state.as_deref(),
+        )
+        .into_response();
+    }
+
+    let code_challenge = match &params.code_challenge {
+        Some(challenge) if !challenge.is_empty() => challenge.as_str(),
+        _ => {
+            return OAuthErrorRedirect::new(
+                &params.redirect_uri,
+                "invalid_request",
+                Some("PKCE required: code_challenge parameter is mandatory"),
+                params.state.as_deref(),
+            )
+            .into_response();
+        }
+    };
+
+    let code_challenge_method = params.code_challenge_method.as_deref().unwrap_or("S256");
+
+    if code_challenge_method != "S256" {
+        return OAuthErrorRedirect::new(
+            &params.redirect_uri,
+            "invalid_request",
+            Some("Invalid code_challenge_method: only 'S256' is supported"),
+            params.state.as_deref(),
+        )
+        .into_response();
+    }
+
+    let resource = match &params.resource {
+        Some(r) => r,
+        None => {
+            return OAuthErrorRedirect::new(
+                &params.redirect_uri,
+                "invalid_request",
+                Some("Missing 'resource' parameter. Required for MCP audience binding."),
+                params.state.as_deref(),
+            )
+            .into_response();
+        }
+    };
+
+    let base_url = BASE_URL.read().await;
+    let frontend_url = format!(
+        "{}/oauth/mcp_authorize?{}",
+        base_url,
+        serde_urlencoded::to_string(&[
+            ("workspace_id", workspace_id.as_str()),
+            ("client_id", params.client_id.as_str()),
+            ("client_name", client.client_name.as_str()),
+            ("redirect_uri", params.redirect_uri.as_str()),
+            ("scope", params.scope.as_deref().unwrap_or("mcp:all")),
+            ("state", params.state.as_deref().unwrap_or("")),
+            ("code_challenge", code_challenge),
+            ("code_challenge_method", code_challenge_method),
+            ("resource", resource),
+        ])
+        .unwrap_or_default()
+    );
+
+    Redirect::temporary(&frontend_url).into_response()
+}
+
+/// POST /api/w/:workspace_id/mcp/oauth/server/approve - user approval (frontend)
+pub async fn workspaced_oauth_approve(
+    Extension(db): Extension<DB>,
+    Path(workspace_id): Path<String>,
+    authed: ApiAuthed,
+    Json(form): Json<ApprovalForm>,
+) -> Result<Json<ApprovalResponse>> {
+    // Verify user is a member of the workspace
+    let is_member = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
+        workspace_id,
+        authed.email
+    )
+    .fetch_one(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
+    .unwrap_or(false);
+
+    if !is_member {
+        return Err(Error::NotAuthorized(
+            "User is not a member of this workspace".to_string(),
+        ));
+    }
+
+    // Verify client exists and redirect_uri is registered
+    let client = sqlx::query_as!(
+        OAuthClient,
+        "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+        form.client_id
+    )
+    .fetch_optional(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
+    .ok_or_else(|| Error::BadRequest("Unknown client_id".to_string()))?;
+
+    if !client.redirect_uris.contains(&form.redirect_uri) {
+        return Err(Error::BadRequest(
+            "Invalid redirect_uri for this client".to_string(),
+        ));
+    }
+
+    if form.code_challenge.is_empty() {
+        return Err(Error::BadRequest(
+            "PKCE required: code_challenge is mandatory".to_string(),
+        ));
+    }
+
+    if form.code_challenge_method.is_empty() {
+        return Err(Error::BadRequest(
+            "PKCE required: code_challenge_method is mandatory".to_string(),
+        ));
+    }
+
+    if form.code_challenge_method != "S256" {
+        return Err(Error::BadRequest(
+            "Invalid code_challenge_method: only 'S256' is supported".to_string(),
+        ));
+    }
+
+    let code = format!("mcp-code-{}", rd_string(32));
+
+    let scopes: Vec<String> = form
+        .scope
+        .split_whitespace()
+        .map(|s| s.to_string())
+        .collect();
+
+    sqlx::query!(
+        "INSERT INTO mcp_oauth_server_code
+         (code, client_id, user_email, workspace_id, scopes, redirect_uri, code_challenge, code_challenge_method)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
+        code,
+        form.client_id,
+        authed.email,
+        workspace_id,
+        &scopes,
+        form.redirect_uri,
+        &form.code_challenge,
+        &form.code_challenge_method,
+    )
+    .execute(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Failed to store authorization code: {}", e)))?;
+
+    Ok(Json(ApprovalResponse {
+        code,
+        state: if form.state.is_empty() {
+            None
+        } else {
+            Some(form.state)
+        },
+    }))
+}
+
+/// PKCE validation (S256 only)
+fn validate_pkce_s256(verifier: &str, challenge: &str) -> bool {
+    let mut hasher = Sha256::new();
+    hasher.update(verifier.as_bytes());
+    let computed = base64_url_encode(&hasher.finalize());
+    constant_time_eq(computed.as_bytes(), challenge.as_bytes())
+}
+
+/// Base64 URL encoding (no padding)
+fn base64_url_encode(data: &[u8]) -> String {
+    use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
+    URL_SAFE_NO_PAD.encode(data)
+}
+
+/// Constant-time comparison
+fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    a.iter().zip(b.iter()).fold(0, |acc, (x, y)| acc | (x ^ y)) == 0
+}
+
+/// Helper for OAuth error redirects
+struct OAuthErrorRedirect {
+    redirect_uri: String,
+    error: String,
+    error_description: Option<String>,
+    state: Option<String>,
+}
+
+impl OAuthErrorRedirect {
+    fn new(
+        redirect_uri: &str,
+        error: &str,
+        error_description: Option<&str>,
+        state: Option<&str>,
+    ) -> Self {
+        Self {
+            redirect_uri: redirect_uri.to_string(),
+            error: error.to_string(),
+            error_description: error_description.map(|s| s.to_string()),
+            state: state.map(|s| s.to_string()),
+        }
+    }
+}
+
+impl IntoResponse for OAuthErrorRedirect {
+    fn into_response(self) -> axum::response::Response {
+        let mut url = format!("{}?error={}", self.redirect_uri, self.error);
+        if let Some(desc) = &self.error_description {
+            url.push_str(&format!("&error_description={}", urlencoding::encode(desc)));
+        }
+        if let Some(state) = &self.state {
+            url.push_str(&format!("&state={}", state));
+        }
+        Redirect::temporary(&url).into_response()
+    }
+}
+
+/// Mounted at /api/mcp/oauth/server
+pub fn global_service() -> Router {
+    Router::new().route("/register", post(oauth_register))
+}
+
+/// Workspace-scoped OAuth endpoints that don't require authentication
+/// Mounted at /api/w/:workspace_id/mcp/oauth/server (outside authenticated section)
+pub fn workspaced_unauthed_service() -> Router {
+    Router::new()
+        .route("/authorize", get(workspaced_oauth_authorize))
+        .route("/token", post(oauth_token))
+}
+
+/// Workspace-scoped OAuth endpoints that require authentication
+/// Mounted at /api/w/:workspace_id/mcp/oauth/server (inside authenticated section)
+pub fn workspaced_authed_service() -> Router {
+    Router::new().route("/approve", post(workspaced_oauth_approve))
+}

backend/windmill-api/src/npm_proxy.rs

@@ -0,0 +1,502 @@
+/*
+ * This file provides a proxy endpoint for npm package requests
+ * to support private registries in the frontend ATA (Automatic Type Acquisition)
+ */
+
+use axum::{
+    extract::{Path, Query},
+    routing::get,
+    Extension, Json, Router,
+};
+use serde::{Deserialize, Serialize};
+use sqlx::types::JsonValue;
+use std::collections::HashMap;
+use tower_http::cors::{Any, CorsLayer};
+use windmill_common::{
+    error::{Error, JsonResult, Result},
+    global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
+    utils::StripPath,
+};
+
+use crate::{db::ApiAuthed, HTTP_CLIENT};
+
+#[derive(Deserialize)]
+struct ProxyQuery {
+    tag: Option<String>,
+}
+
+/// Parse a scoped package path like "@scope/name" or "name" from a wildcard path
+fn parse_package_name(path: &str) -> String {
+    // Remove leading slash if present
+    path.trim_start_matches('/').to_string()
+}
+
+/// Parse package and version from a path like "@scope/name/1.0.0" or "name/1.0.0"
+fn parse_package_and_version(path: &str) -> Result<(String, String)> {
+    let path = path.trim_start_matches('/');
+
+    if path.starts_with('@') {
+        // Scoped package: @scope/name/version
+        let parts: Vec<&str> = path.splitn(3, '/').collect();
+        if parts.len() < 3 {
+            return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string()));
+        }
+        let package = format!("{}/{}", parts[0], parts[1]);
+        let version = parts[2].to_string();
+        Ok((package, version))
+    } else {
+        // Regular package: name/version
+        let parts: Vec<&str> = path.splitn(2, '/').collect();
+        if parts.len() < 2 {
+            return Err(Error::BadRequest("Invalid package path, expected name/version".to_string()));
+        }
+        Ok((parts[0].to_string(), parts[1].to_string()))
+    }
+}
+
+/// Parse package, version, and filepath from a path like "@scope/name/1.0.0/index.d.ts"
+fn parse_package_version_and_file(path: &str) -> Result<(String, String, String)> {
+    let path = path.trim_start_matches('/');
+
+    if path.starts_with('@') {
+        // Scoped package: @scope/name/version/filepath
+        let parts: Vec<&str> = path.splitn(4, '/').collect();
+        if parts.len() < 4 {
+            return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string()));
+        }
+        let package = format!("{}/{}", parts[0], parts[1]);
+        let version = parts[2].to_string();
+        let filepath = parts[3].to_string();
+        Ok((package, version, filepath))
+    } else {
+        // Regular package: name/version/filepath
+        let parts: Vec<&str> = path.splitn(3, '/').collect();
+        if parts.len() < 3 {
+            return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string()));
+        }
+        Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string()))
+    }
+}
+
+#[derive(Serialize)]
+struct PackageVersions {
+    tags: HashMap<String, String>,
+    versions: Vec<String>,
+}
+
+#[derive(Serialize)]
+struct PackageVersion {
+    version: Option<String>,
+}
+
+#[derive(Serialize)]
+struct PackageFiletree {
+    default: String,
+    files: Vec<FileEntry>,
+}
+
+#[derive(Serialize)]
+struct FileEntry {
+    name: String,
+}
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        // Use wildcards for package names to support scoped packages like @scope/package
+        .route("/metadata/*package", get(get_package_metadata))
+        .route("/resolve/*package", get(resolve_package_version))
+        .route("/filetree/*package_version", get(get_package_filetree))
+        .route("/file/*package_version_filepath", get(get_package_file))
+        .layer(
+            CorsLayer::new()
+                .allow_origin(Any)
+                .allow_methods(Any)
+                .allow_headers(Any),
+        )
+}
+
+/// Get package metadata (versions and tags) from the private registry
+async fn get_package_metadata(
+    _authed: ApiAuthed,
+    Path((_w_id, package_path)): Path<(String, StripPath)>,
+    Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
+) -> JsonResult<PackageVersions> {
+    let package = parse_package_name(package_path.to_path());
+    let npm_registry = get_npm_registry(&db).await?;
+
+    if npm_registry.is_none() {
+        return Err(Error::BadRequest(
+            "No private npm registry configured".to_string(),
+        ));
+    }
+
+    let registry_url = npm_registry.unwrap();
+    let package_url = format_registry_url(&registry_url, &package, None, None);
+
+    tracing::info!("Fetching package metadata from: {}", package_url);
+
+    let response = HTTP_CLIENT
+        .get(&package_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
+
+    if !response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Package {} not found in private registry",
+            package
+        )));
+    }
+
+    let package_json: JsonValue = response
+        .json()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
+
+    // Extract versions and dist-tags from the package metadata
+    let mut versions = Vec::new();
+    let mut tags = HashMap::new();
+
+    if let Some(versions_obj) = package_json.get("versions").and_then(|v| v.as_object()) {
+        versions = versions_obj.keys().cloned().collect();
+    }
+
+    if let Some(tags_obj) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
+        for (tag, version) in tags_obj {
+            if let Some(version_str) = version.as_str() {
+                tags.insert(tag.clone(), version_str.to_string());
+            }
+        }
+    }
+
+    Ok(Json(PackageVersions { tags, versions }))
+}
+
+/// Resolve a package tag/version reference to a specific version
+async fn resolve_package_version(
+    _authed: ApiAuthed,
+    Path((_w_id, package_path)): Path<(String, StripPath)>,
+    Query(query): Query<ProxyQuery>,
+    Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
+) -> JsonResult<PackageVersion> {
+    let package = parse_package_name(package_path.to_path());
+    let npm_registry = get_npm_registry(&db).await?;
+
+    if npm_registry.is_none() {
+        return Err(Error::BadRequest(
+            "No private npm registry configured".to_string(),
+        ));
+    }
+
+    let registry_url = npm_registry.unwrap();
+    let reference = query.tag.unwrap_or_else(|| "latest".to_string());
+    let package_url = format_registry_url(&registry_url, &package, None, None);
+
+    tracing::info!("Resolving package version from: {}", package_url);
+
+    let response = HTTP_CLIENT
+        .get(&package_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
+
+    if !response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Package {} not found in private registry",
+            package
+        )));
+    }
+
+    let package_json: JsonValue = response
+        .json()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
+
+    // Try to resolve the reference as a tag first
+    let version = if let Some(tags) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
+        if let Some(version) = tags.get(&reference).and_then(|v| v.as_str()) {
+            Some(version.to_string())
+        } else {
+            // If not a tag, check if it's a valid version
+            if let Some(versions) = package_json.get("versions").and_then(|v| v.as_object()) {
+                if versions.contains_key(&reference) {
+                    Some(reference)
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        }
+    } else {
+        None
+    };
+
+    Ok(Json(PackageVersion { version }))
+}
+
+/// Get the file tree for a specific package version
+async fn get_package_filetree(
+    _authed: ApiAuthed,
+    Path((_w_id, package_version_path)): Path<(String, StripPath)>,
+    Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
+) -> JsonResult<PackageFiletree> {
+    let (package, version) = parse_package_and_version(package_version_path.to_path())?;
+    let npm_registry = get_npm_registry(&db).await?;
+
+    if npm_registry.is_none() {
+        return Err(Error::BadRequest(
+            "No private npm registry configured".to_string(),
+        ));
+    }
+
+    let registry_url = npm_registry.unwrap();
+    let package_url = format_registry_url(&registry_url, &package, None, None);
+
+    tracing::info!("Fetching package filetree from: {}", package_url);
+
+    let response = HTTP_CLIENT
+        .get(&package_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
+
+    if !response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Package {} not found in private registry",
+            package
+        )));
+    }
+
+    let package_json: JsonValue = response
+        .json()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
+
+    // Get the tarball URL for this version
+    let tarball_url = package_json
+        .get("versions")
+        .and_then(|v| v.get(&version))
+        .and_then(|v| v.get("dist"))
+        .and_then(|d| d.get("tarball"))
+        .and_then(|t| t.as_str())
+        .ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
+
+    // Download and extract tarball to get file list
+    let tarball_response = HTTP_CLIENT
+        .get(tarball_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
+
+    if !tarball_response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Failed to download tarball for {}@{}",
+            package, version
+        )));
+    }
+
+    let tarball_bytes = tarball_response
+        .bytes()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
+
+    // Extract file list from tarball
+    let files = extract_tarball_files(&tarball_bytes)?;
+
+    // Find the main entry point
+    let main = package_json
+        .get("versions")
+        .and_then(|v| v.get(&version))
+        .and_then(|v| v.get("main"))
+        .and_then(|m| m.as_str())
+        .unwrap_or("index.js")
+        .to_string();
+
+    Ok(Json(PackageFiletree {
+        default: main,
+        files,
+    }))
+}
+
+/// Get a specific file from a package version
+async fn get_package_file(
+    _authed: ApiAuthed,
+    Path((_w_id, full_path)): Path<(String, StripPath)>,
+    Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
+) -> Result<String> {
+    let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
+    let npm_registry = get_npm_registry(&db).await?;
+
+    if npm_registry.is_none() {
+        return Err(Error::BadRequest(
+            "No private npm registry configured".to_string(),
+        ));
+    }
+
+    let registry_url = npm_registry.unwrap();
+    let package_url = format_registry_url(&registry_url, &package, None, None);
+
+    tracing::info!("Fetching package file from: {}", package_url);
+
+    let response = HTTP_CLIENT
+        .get(&package_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
+
+    if !response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Package {} not found in private registry",
+            package
+        )));
+    }
+
+    let package_json: JsonValue = response
+        .json()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
+
+    // Get the tarball URL for this version
+    let tarball_url = package_json
+        .get("versions")
+        .and_then(|v| v.get(&version))
+        .and_then(|v| v.get("dist"))
+        .and_then(|d| d.get("tarball"))
+        .and_then(|t| t.as_str())
+        .ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
+
+    // Download tarball
+    let tarball_response = HTTP_CLIENT
+        .get(tarball_url)
+        .send()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
+
+    if !tarball_response.status().is_success() {
+        return Err(Error::NotFound(format!(
+            "Failed to download tarball for {}@{}",
+            package, version
+        )));
+    }
+
+    let tarball_bytes = tarball_response
+        .bytes()
+        .await
+        .map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
+
+    // Extract the specific file from the tarball
+    let file_content = extract_file_from_tarball(&tarball_bytes, &filepath)?;
+
+    Ok(file_content)
+}
+
+/// Get the npm registry URL from global settings
+async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
+    let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
+        .await?
+        .and_then(|v| v.as_str().map(|s| s.to_string()));
+
+    Ok(registry)
+}
+
+/// Format a registry URL for a package
+fn format_registry_url(
+    registry_base: &str,
+    package: &str,
+    version: Option<&str>,
+    file: Option<&str>,
+) -> String {
+    let registry_base = registry_base.trim_end_matches('/');
+
+    // Handle scoped packages (e.g., @types/node)
+    let package_path = if package.starts_with('@') {
+        // Scoped packages need to be URL encoded properly
+        package.replace('/', "%2F")
+    } else {
+        package.to_string()
+    };
+
+    match (version, file) {
+        (Some(v), Some(f)) => {
+            format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f)
+        }
+        (Some(v), None) => {
+            format!("{}/{}/{}", registry_base, package_path, v)
+        }
+        _ => {
+            format!("{}/{}", registry_base, package_path)
+        }
+    }
+}
+
+/// Extract file list from a tarball
+fn extract_tarball_files(tarball_bytes: &[u8]) -> Result<Vec<FileEntry>> {
+    use flate2::read::GzDecoder;
+    use tar::Archive;
+
+    let gz = GzDecoder::new(tarball_bytes);
+    let mut archive = Archive::new(gz);
+
+    let mut files = Vec::new();
+
+    for entry in archive
+        .entries()
+        .map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
+    {
+        let entry = entry
+            .map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
+        let path = entry
+            .path()
+            .map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
+
+        // Remove the package/ prefix that npm tarballs have
+        let path_str = path.to_string_lossy().to_string();
+        if let Some(stripped) = path_str.strip_prefix("package/") {
+            files.push(FileEntry {
+                name: format!("/{}", stripped),
+            });
+        }
+    }
+
+    Ok(files)
+}
+
+/// Extract a specific file from a tarball
+fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result<String> {
+    use flate2::read::GzDecoder;
+    use std::io::Read;
+    use tar::Archive;
+
+    let gz = GzDecoder::new(tarball_bytes);
+    let mut archive = Archive::new(gz);
+
+    // Normalize the target file path
+    let target = target_file.trim_start_matches('/');
+
+    for entry in archive
+        .entries()
+        .map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
+    {
+        let mut entry = entry
+            .map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
+        let path = entry
+            .path()
+            .map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
+
+        let path_str = path.to_string_lossy().to_string();
+
+        // Check if this is the file we're looking for
+        if let Some(stripped) = path_str.strip_prefix("package/") {
+            if stripped == target {
+                let mut content = String::new();
+                entry
+                    .read_to_string(&mut content)
+                    .map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?;
+                return Ok(content);
+            }
+        }
+    }
+
+    Err(Error::NotFound(format!("File {} not found in tarball", target_file)))
+}

backend/windmill-api/src/resources.rs

@@ -10,6 +10,7 @@ use std::collections::HashMap;
 
 use crate::{
     db::{ApiAuthed, DB},
+    secret_backend_ext::rename_vault_secret,
     users::{maybe_refresh_folders, require_owner_of_path, Tokened},
     utils::{check_scopes, require_super_admin, BulkDeleteRequest},
     var_resource_cache::{cache_resource, get_cached_resource},
@@ -947,12 +948,40 @@ async fn update_resource(
 
     let mut tx = user_db.begin(&authed).await?;
 
-    if let Some(npath) = ns.path {
+    if let Some(npath) = ns.path.clone() {
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
 
             require_owner_of_path(&authed, path)?;
 
+            // Handle Vault secret rename if the linked variable is a Vault-stored secret
+            let linked_var = sqlx::query!(
+                "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+
+            if let Some(var) = linked_var {
+                if var.is_secret {
+                    // Check if this is a Vault-stored secret and rename it
+                    if let Some(new_value) =
+                        rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
+                    {
+                        // Update the variable's value to point to the new Vault path
+                        sqlx::query!(
+                            "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+                            new_value,
+                            path,
+                            w_id
+                        )
+                        .execute(&mut *tx)
+                        .await?;
+                    }
+                }
+            }
+
             sqlx::query!(
                 "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
                 npath,

backend/windmill-api/src/scripts.rs

@@ -49,7 +49,8 @@ use windmill_common::{
     s3_helpers::upload_artifact_to_store,
     scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline},
     utils::{paginate_without_limits, WarnAfterExt},
-    worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2},
+    min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2},
+    worker::CLOUD_HOSTED,
 };
 
 use windmill_common::{
@@ -2300,12 +2301,12 @@ async fn delete_scripts_bulk(
 /// Validates that script debouncing configuration is supported by all workers
 /// Returns an error if debouncing is configured but workers are behind required version
 async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> {
-    if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await && !ns.debouncing_settings.is_default() {
+    if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await && !ns.debouncing_settings.is_default() {
         tracing::warn!(
             "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature"
         );
         Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() })
-    } else if !*MIN_VERSION_SUPPORTS_DEBOUNCING_V2.read().await
+    } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
         && !ns.debouncing_settings.is_legacy_compatible()
         && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT
     {

backend/windmill-api/src/secret_backend_ext.rs

@@ -0,0 +1,408 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+//! Secret backend extension for the API layer
+//!
+//! This module provides helper functions for integrating the SecretBackend
+//! trait with variable operations in the API.
+//!
+//! Note: HashiCorp Vault integration requires Enterprise Edition.
+//! The OSS version only supports the database backend.
+
+use std::sync::Arc;
+
+use windmill_common::{
+    db::DB,
+    error::{Error, Result},
+    secret_backend::{database::DatabaseBackend, SecretBackend},
+    variables::{build_crypt, decrypt, encrypt},
+};
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use windmill_common::{
+    global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING},
+    secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings},
+};
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use tokio::sync::RwLock;
+
+// Cached Vault backend to avoid recreating it for every request
+// This enables connection pooling and avoids repeated setup overhead
+#[cfg(all(feature = "private", feature = "enterprise"))]
+struct CachedVaultBackend {
+    backend: Arc<dyn SecretBackend>,
+    settings: VaultSettings,
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+lazy_static::lazy_static! {
+    static ref VAULT_BACKEND_CACHE: RwLock<Option<CachedVaultBackend>> = RwLock::new(None);
+}
+
+/// Get the current secret backend based on global settings
+///
+/// OSS: Always returns DatabaseBackend
+/// EE: Returns configured backend (Database or Vault)
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
+    Ok(Arc::new(DatabaseBackend::new(db.clone())))
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
+    let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
+        Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
+        None => SecretBackendConfig::default(),
+    };
+
+    match config {
+        SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))),
+        SecretBackendConfig::HashiCorpVault(settings) => {
+            get_or_create_vault_backend(db, settings).await
+        }
+    }
+}
+
+/// Get a cached Vault backend or create a new one if settings changed
+#[cfg(all(feature = "private", feature = "enterprise"))]
+async fn get_or_create_vault_backend(
+    _db: &DB,
+    settings: VaultSettings,
+) -> Result<Arc<dyn SecretBackend>> {
+    // Check if we have a cached backend with matching settings (read lock)
+    {
+        let cache = VAULT_BACKEND_CACHE.read().await;
+        if let Some(ref cached) = *cache {
+            if cached.settings == settings {
+                return Ok(cached.backend.clone());
+            }
+        }
+    }
+
+    // Need to create a new backend - acquire write lock
+    let mut cache = VAULT_BACKEND_CACHE.write().await;
+
+    // Double-check (another task may have created it while we waited)
+    if let Some(ref cached) = *cache {
+        if cached.settings == settings {
+            return Ok(cached.backend.clone());
+        }
+    }
+
+    // Create new backend
+    let backend: Arc<dyn SecretBackend> = {
+        #[cfg(feature = "openidconnect")]
+        if settings.token.is_none() {
+            Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone()))
+        } else {
+            Arc::new(VaultBackend::new(settings.clone()))
+        }
+
+        #[cfg(not(feature = "openidconnect"))]
+        Arc::new(VaultBackend::new(settings.clone()))
+    };
+
+    // Cache it
+    *cache = Some(CachedVaultBackend {
+        backend: backend.clone(),
+        settings,
+    });
+
+    Ok(backend)
+}
+
+/// Check if a Vault backend is currently configured
+///
+/// OSS: Always returns false
+/// EE: Checks global settings
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn is_vault_backend_configured(_db: &DB) -> Result<bool> {
+    Ok(false)
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn is_vault_backend_configured(db: &DB) -> Result<bool> {
+    let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
+        Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
+        None => SecretBackendConfig::default(),
+    };
+
+    Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_)))
+}
+
+/// Get a secret value using the configured backend
+///
+/// For database backend: decrypts using workspace key
+/// For vault backend (EE only): fetches from Vault directly
+pub async fn get_secret_value(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+    encrypted_value: &str,
+) -> Result<String> {
+    let backend = get_secret_backend(db).await?;
+
+    match backend.backend_name() {
+        "database" => {
+            // Use existing database decryption
+            let mc = build_crypt(db, workspace_id).await?;
+            decrypt(&mc, encrypted_value.to_string()).map_err(|e| {
+                Error::internal_err(format!("Error decrypting variable {}: {}", path, e))
+            })
+        }
+        "hashicorp_vault" => {
+            // Fetch from Vault directly
+            backend.get_secret(workspace_id, path).await
+        }
+        _ => Err(Error::internal_err(format!(
+            "Unknown backend: {}",
+            backend.backend_name()
+        ))),
+    }
+}
+
+/// Store a secret value using the configured backend
+///
+/// For database backend: encrypts using workspace key and returns encrypted value
+/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage
+pub async fn store_secret_value(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+    plain_value: &str,
+) -> Result<String> {
+    let backend = get_secret_backend(db).await?;
+
+    match backend.backend_name() {
+        "database" => {
+            // Use existing database encryption
+            let mc = build_crypt(db, workspace_id).await?;
+            Ok(encrypt(&mc, plain_value))
+        }
+        "hashicorp_vault" => {
+            // Store in Vault and return a marker for DB
+            backend.set_secret(workspace_id, path, plain_value).await?;
+            // Return a marker indicating the value is stored in Vault
+            // The actual value in the DB will be this marker
+            Ok(format!("$vault:{}", path))
+        }
+        _ => Err(Error::internal_err(format!(
+            "Unknown backend: {}",
+            backend.backend_name()
+        ))),
+    }
+}
+
+/// Delete a secret from the configured backend (if using Vault)
+///
+/// For database backend: no-op (DB delete is handled separately)
+/// For vault backend (EE only): deletes from Vault
+pub async fn delete_secret_from_backend(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+) -> Result<()> {
+    if is_vault_backend_configured(db).await? {
+        let backend = get_secret_backend(db).await?;
+        // Ignore NotFound errors during deletion (secret might not exist in Vault)
+        match backend.delete_secret(workspace_id, path).await {
+            Ok(()) => Ok(()),
+            Err(Error::NotFound(_)) => Ok(()),
+            Err(e) => Err(e),
+        }
+    } else {
+        Ok(())
+    }
+}
+
+/// Check if a value is stored in Vault (indicated by the $vault: prefix)
+pub fn is_vault_stored_value(value: &str) -> bool {
+    value.starts_with("$vault:")
+}
+
+/// Rename a secret in Vault when a variable path changes (EE only)
+///
+/// This function:
+/// 1. Reads the secret value from the old path
+/// 2. Writes it to the new path
+/// 3. Deletes from the old path
+/// 4. Returns the new marker value ($vault:new_path)
+///
+/// If the value is not a Vault-stored value, returns None (no action needed).
+/// If Vault is not configured, returns None.
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn rename_vault_secret(
+    _db: &DB,
+    _workspace_id: &str,
+    _old_path: &str,
+    new_path: &str,
+    current_value: &str,
+) -> Result<Option<String>> {
+    // OSS: If value has $vault: prefix, just update the reference
+    // (This handles edge case where EE was used before downgrading to OSS)
+    if is_vault_stored_value(current_value) {
+        tracing::warn!(
+            "Variable has $vault: prefix but Vault requires Enterprise Edition. \
+             Updating DB reference to {}",
+            new_path
+        );
+        return Ok(Some(format!("$vault:{}", new_path)));
+    }
+    Ok(None)
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn rename_vault_secret(
+    db: &DB,
+    workspace_id: &str,
+    old_path: &str,
+    new_path: &str,
+    current_value: &str,
+) -> Result<Option<String>> {
+    // Only handle Vault-stored values
+    if !is_vault_stored_value(current_value) {
+        return Ok(None);
+    }
+
+    // Check if Vault backend is configured
+    if !is_vault_backend_configured(db).await? {
+        // Vault not configured but value has $vault: prefix - this is an inconsistent state
+        // Log warning and return new marker to at least update the DB reference
+        tracing::warn!(
+            "Variable value has $vault: prefix but Vault is not configured. \
+             Updating DB reference from {} to {}",
+            old_path,
+            new_path
+        );
+        return Ok(Some(format!("$vault:{}", new_path)));
+    }
+
+    let backend = get_secret_backend(db).await?;
+
+    // Read from old path
+    let secret_value = match backend.get_secret(workspace_id, old_path).await {
+        Ok(value) => value,
+        Err(Error::NotFound(_)) => {
+            // Secret doesn't exist in Vault - just update the DB reference
+            tracing::warn!(
+                "Secret not found in Vault at path {} during rename to {}",
+                old_path,
+                new_path
+            );
+            return Ok(Some(format!("$vault:{}", new_path)));
+        }
+        Err(e) => return Err(e),
+    };
+
+    // Write to new path
+    backend
+        .set_secret(workspace_id, new_path, &secret_value)
+        .await?;
+
+    // Delete from old path (ignore errors - new path is already written)
+    if let Err(e) = backend.delete_secret(workspace_id, old_path).await {
+        tracing::warn!(
+            "Failed to delete old secret at {} after rename to {}: {}",
+            old_path,
+            new_path,
+            e
+        );
+    }
+
+    Ok(Some(format!("$vault:{}", new_path)))
+}
+
+/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename)
+/// EE only feature.
+///
+/// This is used when renaming users where many secrets need their paths updated.
+/// Returns a list of (old_path, new_value) pairs for updating the database.
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn rename_vault_secrets_with_prefix(
+    _db: &DB,
+    _workspace_id: &str,
+    _old_prefix: &str,
+    _new_prefix: &str,
+    _variables: Vec<(String, String)>,
+) -> Result<Vec<(String, String)>> {
+    // OSS: No Vault support, return empty
+    Ok(vec![])
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn rename_vault_secrets_with_prefix(
+    db: &DB,
+    workspace_id: &str,
+    old_prefix: &str,
+    new_prefix: &str,
+    variables: Vec<(String, String)>, // (path, value) pairs
+) -> Result<Vec<(String, String)>> {
+    // Only process if Vault is configured
+    if !is_vault_backend_configured(db).await? {
+        return Ok(vec![]);
+    }
+
+    let backend = get_secret_backend(db).await?;
+    let mut updates = Vec::new();
+
+    for (old_path, value) in variables {
+        // Only handle Vault-stored values
+        if !is_vault_stored_value(&value) {
+            continue;
+        }
+
+        // Calculate new path by replacing prefix
+        let new_path = if old_path.starts_with(old_prefix) {
+            format!("{}{}", new_prefix, &old_path[old_prefix.len()..])
+        } else {
+            continue; // Path doesn't match prefix, skip
+        };
+
+        // Read from old path
+        let secret_value = match backend.get_secret(workspace_id, &old_path).await {
+            Ok(v) => v,
+            Err(Error::NotFound(_)) => {
+                // Just update DB reference
+                updates.push((old_path, format!("$vault:{}", new_path)));
+                continue;
+            }
+            Err(e) => {
+                tracing::error!(
+                    "Failed to read secret at {} during bulk rename: {}",
+                    old_path,
+                    e
+                );
+                continue;
+            }
+        };
+
+        // Write to new path
+        if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await {
+            tracing::error!(
+                "Failed to write secret to {} during bulk rename: {}",
+                new_path,
+                e
+            );
+            continue;
+        }
+
+        // Delete from old path
+        if let Err(e) = backend.delete_secret(workspace_id, &old_path).await {
+            tracing::warn!(
+                "Failed to delete old secret at {} after rename: {}",
+                old_path,
+                e
+            );
+        }
+
+        updates.push((old_path, format!("$vault:{}", new_path)));
+    }
+
+    Ok(updates)
+}

backend/windmill-api/src/settings.rs

@@ -31,6 +31,8 @@ use crate::utils::require_devops_role;
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "enterprise")]
 use windmill_common::ee_oss::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings};
 use windmill_common::{
     email_oss::send_email_plain_text,
     error::{self, JsonResult, Result},
@@ -85,6 +87,16 @@ pub fn global_service() -> Router {
             post(acknowledge_all_critical_alerts),
         );
 
+    // Vault integration routes (EE only - requires both private and enterprise features)
+    #[cfg(all(feature = "private", feature = "enterprise"))]
+    let r = r
+        .route("/test_secret_backend", post(test_secret_backend))
+        .route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault))
+        .route(
+            "/migrate_secrets_to_database",
+            post(migrate_secrets_to_database),
+        );
+
     #[cfg(feature = "parquet")]
     {
         return r.route("/test_object_storage_config", post(test_s3_bucket));
@@ -736,7 +748,11 @@ async fn setup_custom_instance_pg_database_inner(
             "Cannot use reserved PostgreSQL database names".to_string(),
         ));
     }
-    if wmill_pg_creds.dbname.trim().eq_ignore_ascii_case(dbname.trim()) {
+    if wmill_pg_creds
+        .dbname
+        .trim()
+        .eq_ignore_ascii_case(dbname.trim())
+    {
         return Err(error::Error::BadRequest(
             "Database name cannot be the same as the main database".to_string(),
         ));
@@ -751,10 +767,7 @@ async fn setup_custom_instance_pg_database_inner(
     .await?
     .unwrap_or(false);
 
-    let pg_creds = PgDatabase {
-        dbname: dbname.to_string(),
-        ..wmill_pg_creds
-    };
+    let pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds };
 
     logs.created_database = "SKIP".to_string();
     if !db_exists {
@@ -777,7 +790,8 @@ async fn setup_custom_instance_pg_database_inner(
              GRANT CREATE ON SCHEMA public TO custom_instance_user;
              GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user;
              ALTER DEFAULT PRIVILEGES IN SCHEMA public
-                 GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;"
+                 GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;
+             ALTER ROLE custom_instance_user CREATEROLE;"
         ))
         .await
         .map_err(|e| {
@@ -798,3 +812,100 @@ async fn setup_custom_instance_pg_database_inner(
 
     Ok(())
 }
+
+// ============================================================================
+// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition
+// ============================================================================
+
+/// Test connection to a secret backend (HashiCorp Vault)
+///
+/// This endpoint validates that the Vault settings are correct and that
+/// Windmill can successfully authenticate and communicate with Vault.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn test_secret_backend(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?;
+
+    Ok("Successfully connected to HashiCorp Vault".to_string())
+}
+
+/// Migrate existing secrets from database to HashiCorp Vault
+///
+/// This endpoint reads all encrypted secrets from the database, decrypts them,
+/// and stores them in HashiCorp Vault. The database values are NOT deleted
+/// automatically to allow for rollback if needed.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn migrate_secrets_to_vault(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> JsonResult<SecretMigrationReport> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?;
+
+    Ok(Json(report))
+}
+
+/// Migrate secrets from HashiCorp Vault back to database
+///
+/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using
+/// the workspace encryption keys, and stores them in the database. The Vault
+/// values are NOT deleted automatically to allow for rollback if needed.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn migrate_secrets_to_database(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> JsonResult<SecretMigrationReport> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let report =
+        windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?;
+
+    Ok(Json(report))
+}
+
+// ============================================================================
+// JWKS Endpoint for Vault JWT Authentication
+// ============================================================================
+
+/// JSON Web Key Set response structure
+#[derive(Serialize)]
+pub struct JwksResponse {
+    pub keys: Vec<serde_json::Value>,
+}
+
+/// JWKS endpoint for HashiCorp Vault to validate JWTs
+///
+/// Vault calls this endpoint to fetch the public keys used to verify
+/// JWTs generated by Windmill for authentication.
+///
+/// In the open-source version, this returns an empty JWKS.
+/// The Enterprise Edition provides the actual key set.
+pub async fn get_jwks() -> JsonResult<JwksResponse> {
+    // Open source version returns empty JWKS
+    // Enterprise Edition will override this with actual public keys
+    #[cfg(not(feature = "enterprise"))]
+    {
+        Ok(Json(JwksResponse { keys: vec![] }))
+    }
+
+    #[cfg(feature = "enterprise")]
+    {
+        // In enterprise mode, the actual keys would be fetched from global settings
+        // For now, return empty - the EE implementation would override this
+        Ok(Json(JwksResponse { keys: vec![] }))
+    }
+}

backend/windmill-api/src/users.rs

@@ -19,6 +19,7 @@ use crate::db::ApiAuthed;
 
 pub use crate::auth::Tokened;
 
+use crate::secret_backend_ext::rename_vault_secrets_with_prefix;
 use crate::utils::{
     generate_instance_wide_unique_username, get_instance_username_or_create_pending,
 };
@@ -2661,6 +2662,7 @@ async fn rename_user(
         }
         update_username_in_workpsace(
             &mut tx,
+            &db,
             &user_email,
             &w_u.username,
             &ru.new_username,
@@ -2688,6 +2690,7 @@ async fn rename_user(
 
 async fn update_username_in_workpsace<'c>(
     tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    db: &DB,
     email: &str,
     old_username: &str,
     new_username: &str,
@@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>(
 
     // ---- variables ----
 
+    // Handle Vault secret renames before updating paths in DB
+    let old_prefix = format!("u/{}/", old_username);
+    let new_prefix = format!("u/{}/", new_username);
+
+    // Fetch all Vault-stored secret variables under this user's path
+    let vault_secrets: Vec<(String, String)> = sqlx::query!(
+        r#"SELECT path, value FROM variable
+           WHERE path LIKE ('u/' || $1 || '/%')
+           AND workspace_id = $2
+           AND is_secret = true
+           AND value LIKE '$vault:%'"#,
+        old_username,
+        w_id
+    )
+    .fetch_all(&mut **tx)
+    .await?
+    .into_iter()
+    .map(|r| (r.path, r.value))
+    .collect();
+
+    // Rename secrets in Vault and get the new values
+    let vault_updates =
+        rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?;
+
+    // Update the values in the DB for renamed Vault secrets (using OLD path, before path update)
+    for (old_path, new_value) in vault_updates {
+        sqlx::query!(
+            "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+            new_value,
+            old_path,
+            w_id
+        )
+        .execute(&mut **tx)
+        .await?;
+    }
+
+    // Now update the paths in the database
     sqlx::query!(
         r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
         new_username,

backend/windmill-api/src/variables.rs

@@ -8,6 +8,10 @@
 
 use crate::{
     db::{ApiAuthed, DB},
+    secret_backend_ext::{
+        delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret,
+        store_secret_value,
+    },
     users::{maybe_refresh_folders, require_owner_of_path},
     utils::{check_scopes, BulkDeleteRequest},
     webhook_util::{WebhookMessage, WebhookShared},
@@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable};
 use lazy_static::lazy_static;
 use serde::Deserialize;
 use sqlx::{Acquire, Postgres, Transaction};
-use windmill_common::variables::{decrypt, encrypt};
+use windmill_common::variables::encrypt;
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 
 lazy_static! {
@@ -210,13 +214,8 @@ async fn get_variable(
                 return Err(Error::internal_err("Require oauth2 feature".to_string()));
             } else if !value.is_empty() && decrypt_secret {
                 let _ = tx.commit().await;
-                let mc = build_crypt(&db, &w_id).await?;
-                Some(decrypt(&mc, value).map_err(|e| {
-                    Error::internal_err(format!(
-                        "Error decrypting variable {}: {}",
-                        variable.path, e
-                    ))
-                })?)
+                // Use secret backend for decryption (supports both DB and Vault)
+                Some(get_secret_value(&db, &w_id, &variable.path, &value).await?)
             } else if q.include_encrypted.unwrap_or(false) {
                 Some(value)
             } else {
@@ -355,8 +354,8 @@ async fn create_variable(
 
     check_path_conflict(&db, &w_id, &variable.path).await?;
     let value = if variable.is_secret && !already_encrypted.unwrap_or(false) {
-        let mc = build_crypt(&db, &w_id).await?;
-        encrypt(&mc, &variable.value)
+        // Use secret backend for encryption (supports both DB and Vault)
+        store_secret_value(&db, &w_id, &variable.path, &variable.value).await?
     } else {
         variable.value
     };
@@ -436,6 +435,16 @@ async fn delete_variable(
 
     check_scopes(&authed, || format!("variables:write:{}", path))?;
 
+    // Check if variable is a secret before deleting (for Vault cleanup)
+    let is_secret = sqlx::query_scalar!(
+        "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+        path,
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .unwrap_or(false);
+
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
@@ -465,6 +474,11 @@ async fn delete_variable(
 
     tx.commit().await?;
 
+    // If variable was a secret, also delete from Vault backend (if configured)
+    if is_secret {
+        delete_secret_from_backend(&db, &w_id, path).await?;
+    }
+
     handle_deployment_metadata(
         &authed.email,
         &authed.username,
@@ -496,6 +510,15 @@ async fn delete_variables_bulk(
         check_scopes(&authed, || format!("variables:write:{}", path))?;
     }
 
+    // Query which paths are secrets before deletion (for Vault cleanup)
+    let secret_paths: Vec<String> = sqlx::query_scalar!(
+        "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
+        &request.paths,
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
+
     let mut tx = user_db.begin(&authed).await?;
 
     let deleted_paths = sqlx::query_scalar!(
@@ -526,6 +549,13 @@ async fn delete_variables_bulk(
 
     tx.commit().await?;
 
+    // Delete secrets from Vault backend (if configured)
+    for path in &secret_paths {
+        if deleted_paths.contains(path) {
+            delete_secret_from_backend(&db, &w_id, path).await?;
+        }
+    }
+
     try_join_all(deleted_paths.iter().map(|path| {
         handle_deployment_metadata(
             &authed.email,
@@ -589,7 +619,9 @@ async fn update_variable(
         sqlb.set_str("path", npath);
     }
     let ns_value_is_none = ns.value.is_none();
-    if let Some(nvalue) = ns.value {
+    // Determine the target path for storing secrets (use new path if provided)
+    let target_path = ns.path.as_deref().unwrap_or(path);
+    if let Some(nvalue) = ns.value.clone() {
         let is_secret = if ns.is_secret.is_some() {
             ns.is_secret.unwrap()
         } else {
@@ -604,8 +636,9 @@ async fn update_variable(
         };
 
         let value = if is_secret && !already_encrypted.unwrap_or(false) {
-            let mc = build_crypt(&db, &w_id).await?;
-            encrypt(&mc, &nvalue)
+            // Use secret backend for encryption (supports both DB and Vault)
+            // Store at target_path (new path if renaming, otherwise current path)
+            store_secret_value(&db, &w_id, target_path, &nvalue).await?
         } else {
             nvalue
         };
@@ -654,11 +687,38 @@ async fn update_variable(
 
     let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?;
 
-    if let Some(npath) = ns.path {
+    if let Some(npath) = ns.path.clone() {
         if npath != path {
             check_path_conflict(&db, &w_id, &npath).await?;
             require_owner_of_path(&authed, path)?;
 
+            // Handle Vault secret rename if the variable is a secret stored in Vault
+            let current_var = sqlx::query!(
+                "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+
+            if let Some(var) = current_var {
+                if var.is_secret && is_vault_stored_value(&var.value) {
+                    if ns.value.is_some() {
+                        // New value was provided and already stored at new path
+                        // Just delete the old secret from Vault
+                        delete_secret_from_backend(&db, &w_id, path).await?;
+                    } else {
+                        // No new value - rename the secret in Vault
+                        if let Some(new_value) =
+                            rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
+                        {
+                            // Update the variable's value to point to the new Vault path
+                            sqlb.set_str("value", &new_value);
+                        }
+                    }
+                }
+            }
+
             let mut v = sqlx::query_scalar!(
                 "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
                 path,
@@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>(
             #[cfg(not(feature = "oauth2"))]
             return Err(Error::internal_err("Require oauth2 feature".to_string()));
         } else if !value.is_empty() {
-            let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?;
-            decrypt(&mc, value).map_err(|e| {
-                Error::internal_err(format!(
-                    "Error decrypting variable {}: {}",
-                    variable.path, e
-                ))
-            })?
+            // Use secret backend for decryption (supports both DB and Vault)
+            get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await?
         } else {
             "".to_string()
         }

backend/windmill-api/src/workers.rs

@@ -7,7 +7,7 @@
  */
 
 use axum::{
-    extract::{Extension, Query},
+    extract::{Extension, Path, Query},
     routing::get,
     Json, Router,
 };
@@ -18,7 +18,7 @@ use uuid::Uuid;
 use windmill_common::{
     db::UserDB,
     error::JsonResult,
-    jobs::TAGS_ARE_SENSITIVE,
+    jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE},
     utils::{paginate, Pagination},
     worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE},
     DB,
@@ -41,6 +41,10 @@ pub fn global_service() -> Router {
         .route("/queue_running_counts", get(get_queue_running_counts))
 }
 
+pub fn workspaced_service() -> Router {
+    Router::new().route("/custom_tags", get(get_custom_tags_for_workspace))
+}
+
 #[derive(FromRow, Serialize, Deserialize)]
 struct WorkerPing {
     worker: String,
@@ -93,6 +97,9 @@ async fn list_worker_pings(
     Query(query): Query<ListWorkerQuery>,
 ) -> JsonResult<Vec<WorkerPing>> {
     let is_super_admin = require_super_admin(&db, &authed.email).await.is_ok();
+    if *HIDE_WORKERS_FOR_NON_ADMINS && !is_super_admin {
+        return Ok(Json(vec![]));
+    }
     let mut tx = user_db.begin(&authed).await?;
 
     let (per_page, offset) = paginate(Pagination { page: query.page, per_page: query.per_page });
@@ -189,24 +196,15 @@ async fn exists_workers_with_tags(
 
 #[derive(Deserialize)]
 struct CustomTagQuery {
-    workspace: Option<String>,
     show_workspace_restriction: Option<bool>,
 }
+
 async fn get_custom_tags(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Query(query): Query<CustomTagQuery>,
 ) -> JsonResult<Vec<String>> {
-    if query.show_workspace_restriction.is_some_and(|x| x) && query.workspace.is_some() {
-        return Err(windmill_common::error::Error::BadRequest(
-            "Cannot use both workspace and show_workspace_restriction".to_string(),
-        ));
-    }
-    if let Some(workspace) = query.workspace {
-        let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
-        let all_tags = tags_o.to_string_vec(Some(workspace));
-        return Ok(Json(all_tags));
-    } else if query.show_workspace_restriction.is_some_and(|x| x) {
+    if query.show_workspace_restriction.is_some_and(|x| x) {
         let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
         let all_tags = tags_o.to_string_vec(None);
         return Ok(Json(all_tags));
@@ -220,6 +218,15 @@ async fn get_custom_tags(
     Ok(Json(ALL_TAGS.read().await.clone().into()))
 }
 
+async fn get_custom_tags_for_workspace(
+    _authed: ApiAuthed,
+    Path(w_id): Path<String>,
+) -> JsonResult<Vec<String>> {
+    let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
+    let all_tags = tags_o.to_string_vec(Some(w_id));
+    Ok(Json(all_tags))
+}
+
 async fn get_default_tags_per_workspace() -> JsonResult<bool> {
     Ok(Json(
         DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed),