experiment: Add full flow executor with branch/loop support

Add flow_executor.rs that supports executing complex flows in local mode:
- ForloopFlow: iterate over arrays/ranges with sequential execution
- WhileloopFlow: execute modules while condition is true
- BranchOne: if/else branching based on conditions
- BranchAll: parallel branch execution (sequential for now)
- RawScript: inline script execution (bash, python, deno, bun)
- Identity: pass-through module

Key features:
- Uses windmill-common FlowValue types for compatibility
- Expression evaluation for input transforms with comparisons
- Proper flow status tracking with module results
- Recursive async execution with async_recursion crate

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/docker-image.yml

@@ -167,39 +167,6 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-  # disabled until we make it 100% reliable and add more meaningful tests
-  # playwright:
-  #   runs-on: [self-hosted, new]
-  #   needs: [build]
-  #   services:
-  #     postgres:
-  #       image: postgres
-  #       env:
-  #         POSTGRES_DB: windmill
-  #         POSTGRES_USER: admin
-  #         POSTGRES_PASSWORD: changeme
-  #       ports:
-  #         - 5432:5432
-  #       options: >-
-  #         --health-cmd pg_isready
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 5
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: "Docker"
-  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-  #       id: docker-container
-  #     - uses: actions/setup-node@v3
-  #       with:
-  #         node-version: 16
-  #     - name: "Playwright run"
-  #       timeout-minutes: 2
-  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
-  #     - name: "Clean up"
-  #       run: docker kill ${{ steps.docker-container.outputs.id }}
-  #       if: always()
-
   attach_amd64_binary_to_release:
     needs: [build, build_ee]
     runs-on: ubicloud

CHANGELOG.md

@@ -1,5 +1,110 @@
 # Changelog
 
+## [1.613.4](https://github.com/windmill-labs/windmill/compare/v1.613.3...v1.613.4) (2026-01-21)
+
+
+### Bug Fixes
+
+* update git sync CLI to 1.613.2 ([7848d36](https://github.com/windmill-labs/windmill/commit/7848d361a546ef8a2ef9fa20a0399f2e8473f685))
+
+## [1.613.3](https://github.com/windmill-labs/windmill/compare/v1.613.2...v1.613.3) (2026-01-21)
+
+
+### Bug Fixes
+
+* **cli:** normalize paths in wmill-lock for cross-platform compatibility ([#7645](https://github.com/windmill-labs/windmill/issues/7645)) ([8cf456d](https://github.com/windmill-labs/windmill/commit/8cf456d74c79921806edbcd0c9fde462f7202188))
+* update git sync CLI to 1.613.2 ([1e4fe01](https://github.com/windmill-labs/windmill/commit/1e4fe01293e65ac130b368fbef45a1571ee2b6d7))
+
+## [1.613.2](https://github.com/windmill-labs/windmill/compare/v1.613.1...v1.613.2) (2026-01-21)
+
+
+### Bug Fixes
+
+* azure read s3 proxy ([#7641](https://github.com/windmill-labs/windmill/issues/7641)) ([9e617a3](https://github.com/windmill-labs/windmill/commit/9e617a3979622a58ae022f9a74e2dde87a43c60e))
+* **cli:** skip branch-specific files when type is not configured ([#7643](https://github.com/windmill-labs/windmill/issues/7643)) ([287b7e7](https://github.com/windmill-labs/windmill/commit/287b7e7d971469db979e6951ee14764f6b91ed67))
+
+## [1.613.1](https://github.com/windmill-labs/windmill/compare/v1.613.0...v1.613.1) (2026-01-21)
+
+
+### Bug Fixes
+
+* fix microsoft SSO setting ([3f3df41](https://github.com/windmill-labs/windmill/commit/3f3df4163f9b6d99bc2c0f0284134b8b78f1ef6d))
+* isolate SvelteKit-specific imports for library usage ([203f678](https://github.com/windmill-labs/windmill/commit/203f6785c4ba9f7ace643259bf4e4a8f164288f3))
+
+## [1.613.0](https://github.com/windmill-labs/windmill/compare/v1.612.2...v1.613.0) (2026-01-20)
+
+
+### Features
+
+* **api:** add include_args query parameter to job list endpoints ([96dabee](https://github.com/windmill-labs/windmill/commit/96dabee22591adff5d6221e8628f7a1571b8d5a8))
+* **cli:** add workspace list command to show remote workspaces ([a08c52e](https://github.com/windmill-labs/windmill/commit/a08c52ec8f5323c645457b1dd7b32ef703fd86c4))
+* DuckDB support write to Azure ([#7618](https://github.com/windmill-labs/windmill/issues/7618)) ([73e86d9](https://github.com/windmill-labs/windmill/commit/73e86d9fc867aedb7221cf3da9df8ab573734d0f))
+* **mcp:** handle server oauth ([#7585](https://github.com/windmill-labs/windmill/issues/7585)) ([09adc58](https://github.com/windmill-labs/windmill/commit/09adc58a678da2d59d20e27a6b528b79843f122f))
+* otel REST tracing ([#7571](https://github.com/windmill-labs/windmill/issues/7571)) ([95df7b9](https://github.com/windmill-labs/windmill/commit/95df7b9a6a8ffcbca92b3249a61d97c32c9dbc4f))
+* **raw-apps:** add ctx input type for secure backend-resolved user context ([#7621](https://github.com/windmill-labs/windmill/issues/7621)) ([c143e78](https://github.com/windmill-labs/windmill/commit/c143e78d7fdd866b2e30c036ef48e907dda6ad6c))
+* **raw-apps:** add public URL and custom path support for raw apps ([#7630](https://github.com/windmill-labs/windmill/issues/7630)) ([baf060d](https://github.com/windmill-labs/windmill/commit/baf060df7474620b144e4a438274866bdfc41881))
+* **raw-apps:** enable hash-based routing with URL sync for shareable URLs ([#7624](https://github.com/windmill-labs/windmill/issues/7624)) ([3205949](https://github.com/windmill-labs/windmill/commit/32059499d5fa9aed1f8149f427732d1f0500dce5))
+
+
+### Bug Fixes
+
+* **cli:** recognize branch-specific folder files in getTypeStrFromPath ([6f35279](https://github.com/windmill-labs/windmill/commit/6f35279126b875d09af23d4618e86f87be064679))
+* **cli:** recognize branch-specific settings and encryption_key files ([5c1c682](https://github.com/windmill-labs/windmill/commit/5c1c682dcaa1a4ce80ee4de78b80c9ace395092a))
+* **frontend:** improve raw app history  ([#7625](https://github.com/windmill-labs/windmill/issues/7625)) ([687175c](https://github.com/windmill-labs/windmill/commit/687175c6a85f47c707bb429008c93ec0981c50e2))
+* **frontend:** set editor font size to the same default as text ([#7631](https://github.com/windmill-labs/windmill/issues/7631)) ([d884ddb](https://github.com/windmill-labs/windmill/commit/d884ddb7eb611f17a8e0ed41998953fb47b6cc21))
+* S3 advanced custom permissions ([#7632](https://github.com/windmill-labs/windmill/issues/7632)) ([1526d3a](https://github.com/windmill-labs/windmill/commit/1526d3ae2b3139bbfa23aef012bbe7f9b2132732))
+
+## [1.612.2](https://github.com/windmill-labs/windmill/compare/v1.612.1...v1.612.2) (2026-01-19)
+
+
+### Bug Fixes
+
+* add HIDE_WORKERS_FOR_NON_ADMINS env var and workspace-scoped custom_tags endpoint ([#7613](https://github.com/windmill-labs/windmill/issues/7613)) ([f33b799](https://github.com/windmill-labs/windmill/commit/f33b79936b8666242f2235d4fa6d4e7488123194))
+* **mcp:** fix empty args format + sanitize tool name ([#7615](https://github.com/windmill-labs/windmill/issues/7615)) ([f55dac6](https://github.com/windmill-labs/windmill/commit/f55dac69582000f0bfdae6dbed2d33f2e48087b2))
+
+## [1.612.1](https://github.com/windmill-labs/windmill/compare/v1.612.0...v1.612.1) (2026-01-19)
+
+
+### Bug Fixes
+
+* fix runs page initialization ([1438b26](https://github.com/windmill-labs/windmill/commit/1438b263102ccf22612f98e59596c7e51083df71))
+* update git sync CLI to 1.612.0 ([8daeccc](https://github.com/windmill-labs/windmill/commit/8daeccc89fc0405143a2b553520f9a42272e3c26))
+
+## [1.612.0](https://github.com/windmill-labs/windmill/compare/v1.611.0...v1.612.0) (2026-01-19)
+
+
+### Features
+
+* **cli:** add branch-specific items for folders and settings ([#7611](https://github.com/windmill-labs/windmill/issues/7611)) ([3ec9439](https://github.com/windmill-labs/windmill/commit/3ec94395dcc6a179a4d5dde3a5b88aeb1053ada3))
+* move job metrics from ee to ce ([#7608](https://github.com/windmill-labs/windmill/issues/7608)) ([c04eb37](https://github.com/windmill-labs/windmill/commit/c04eb371ccd805e8d0d0a03b4ef654c7a8131ccd))
+
+
+### Bug Fixes
+
+* **frontend:** fix centered page shift when scroll ([#7610](https://github.com/windmill-labs/windmill/issues/7610)) ([c1ec159](https://github.com/windmill-labs/windmill/commit/c1ec159471d3fabb9cb7b9023d662726a9cf1f93))
+* **frontend:** improve ai settings page ([#7606](https://github.com/windmill-labs/windmill/issues/7606)) ([9359ad8](https://github.com/windmill-labs/windmill/commit/9359ad820ded8a4a94195ee76bbe6210f6e8eb9f))
+* **frontend:** improve loading centered modal ui ([#7605](https://github.com/windmill-labs/windmill/issues/7605)) ([30da9e6](https://github.com/windmill-labs/windmill/commit/30da9e69f88ba4621bb3ee35287f914389930bb6))
+
+## [1.611.0](https://github.com/windmill-labs/windmill/compare/v1.610.1...v1.611.0) (2026-01-19)
+
+
+### Features
+
+* add HashiCorp Vault secret storage integration ([#7599](https://github.com/windmill-labs/windmill/issues/7599)) ([1b9d1c5](https://github.com/windmill-labs/windmill/commit/1b9d1c56c7e49042677326eb397e10d34a3ddcdf))
+
+
+### Bug Fixes
+
+* **flow-chat:** handle SSE timeout and fix temp message race condition ([4f8110e](https://github.com/windmill-labs/windmill/commit/4f8110eb9852b78b48aabbed114c75cbf0d1a2ef))
+
+## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17)
+
+
+### Bug Fixes
+
+* resolve BlobPart type incompatibility between Deno and Node.js ([2eac74c](https://github.com/windmill-labs/windmill/commit/2eac74cef4aa5a987fb16110388f99e912951db8))
+* use type cast instead of slice() for BlobPart compatibility ([ff77154](https://github.com/windmill-labs/windmill/commit/ff771546380ef26dfa443f9d459853048bc8029c))
+
 ## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17)
 
 

backend/.sqlx/query-002d68d7c4437522a6dae95af007a356217bbae06b8453f0c32046f0cbf20dcb.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_by",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "002d68d7c4437522a6dae95af007a356217bbae06b8453f0c32046f0cbf20dcb"
+}

backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c"
+}

backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336"
+}

backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path FROM variable WHERE is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088"
+}

backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "TextArray",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d"
+}

backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_secret",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e"
+}

backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "is_secret",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65"
+}

backend/.sqlx/query-1b4f7485c015338536d781838448c96ce686fce217be21ec15a8900b772f02a3.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_server_code\n         (code, client_id, user_email, workspace_id, scopes, redirect_uri, code_challenge, code_challenge_method)\n         VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Text",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1b4f7485c015338536d781838448c96ce686fce217be21ec15a8900b772f02a3"
+}

backend/.sqlx/query-1d8c2f54118b352dc13058dbb9b6e3f6ca4961b68d7e409386e655a61c54e0d0.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1d8c2f54118b352dc13058dbb9b6e3f6ca4961b68d7e409386e655a61c54e0d0"
+}

backend/.sqlx/query-2922c242228b2188b8abcda02b37d6fd220659dcd9e16d4bb110202321bc06cf.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "client_name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "redirect_uris",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "2922c242228b2188b8abcda02b37d6fd220659dcd9e16d4bb110202321bc06cf"
+}

backend/.sqlx/query-2c231a2cd267d8d6d28a22d166a50cc6b4df813a15c613eb1960eff202c517f8.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_refresh_token\n         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)\n         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2c231a2cd267d8d6d28a22d166a50cc6b4df813a15c613eb1960eff202c517f8"
+}

backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value FROM variable\n           WHERE path LIKE ('u/' || $1 || '/%')\n           AND workspace_id = $2\n           AND is_secret = true\n           AND value LIKE '$vault:%'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e"
+}

backend/.sqlx/query-369f8ecde50af034f06d339ecef8fc55a0113b4156274d03d5af643c3da73fa4.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM mcp_oauth_server_code\n         WHERE code = $1 AND expires_at > now()\n         RETURNING code, client_id, user_email, workspace_id, scopes, redirect_uri,\n                   code_challenge, code_challenge_method",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "code",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "user_email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "scopes",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 5,
+        "name": "redirect_uri",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "code_challenge",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "code_challenge_method",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "369f8ecde50af034f06d339ecef8fc55a0113b4156274d03d5af643c3da73fa4"
+}

backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200"
+}

backend/.sqlx/query-5769af6cfc749881b3f21d42d2c79b4c3e6788ba521ef5736f46d6ec8447ad8f.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM mcp_oauth_server_code WHERE expires_at <= now() RETURNING code",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "code",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "5769af6cfc749881b3f21d42d2c79b4c3e6788ba521ef5736f46d6ec8447ad8f"
+}

backend/.sqlx/query-5b5cb5339208847bd542f8d903d028803a286e69443a151cd1e3d16da7e8e4f7.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO otel_traces (\n                trace_id, span_id, trace_state, parent_span_id, flags,\n                name, kind, start_time_unix_nano, end_time_unix_nano,\n                attributes, dropped_attributes_count,\n                events, dropped_events_count,\n                links, dropped_links_count,\n                status\n            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bytea",
+        "Bytea",
+        "Text",
+        "Bytea",
+        "Int4",
+        "Text",
+        "Int4",
+        "Int8",
+        "Int8",
+        "Jsonb",
+        "Int4",
+        "Jsonb",
+        "Int4",
+        "Jsonb",
+        "Int4",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5b5cb5339208847bd542f8d903d028803a286e69443a151cd1e3d16da7e8e4f7"
+}

backend/.sqlx/query-5c9ed4d8d16c77c0c6b42e9ee211168573162745060788fbca188ed405c423cd.json

@@ -0,0 +1,89 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mcp_oauth_refresh_token\n         SET used_at = now()\n         WHERE refresh_token = $1\n           AND client_id = $2\n           AND used_at IS NULL\n           AND NOT revoked\n           AND expires_at > now()\n         RETURNING id, refresh_token, access_token, client_id, user_email, workspace_id,\n                   scopes, token_family, created_at, expires_at, used_at, revoked",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "refresh_token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "access_token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "client_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "user_email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "scopes",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 7,
+        "name": "token_family",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 8,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 9,
+        "name": "expires_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 10,
+        "name": "used_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 11,
+        "name": "revoked",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "5c9ed4d8d16c77c0c6b42e9ee211168573162745060788fbca188ed405c423cd"
+}

backend/.sqlx/query-5ffb1c49d8d001253a71c6b9bd90e58416d59a9a855afd1ec0a814937583461f.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mcp_oauth_refresh_token SET revoked = TRUE WHERE token_family = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5ffb1c49d8d001253a71c6b9bd90e58416d59a9a855afd1ec0a814937583461f"
+}

backend/.sqlx/query-6376f88654dbbd85a68c507480fc4918958244abd7a1f81f32a0e60f7e5f9464.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO mcp_oauth_server_client (client_id, client_name, redirect_uris)\n         VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "TextArray"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6376f88654dbbd85a68c507480fc4918958244abd7a1f81f32a0e60f7e5f9464"
+}

backend/.sqlx/query-66e0968fe9f757755945a7010153821cf73ace9d6692750ccc4cca37701ed77a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM token WHERE token = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "66e0968fe9f757755945a7010153821cf73ace9d6692750ccc4cca37701ed77a"
+}

backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d"
+}

backend/.sqlx/query-7e4aa6b19b110bca423b3a3f428826d92b9808c64ef989fef2142bc8e02d6630.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)\n         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Text",
+        "TextArray",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7e4aa6b19b110bca423b3a3f428826d92b9808c64ef989fef2142bc8e02d6630"
+}

backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a"
+}

backend/.sqlx/query-90d93fd3bd91e468c1e796e41e31e4f15a825b442346c7386836127bc5723c93.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT json_build_object(\n            'trace_id', encode(trace_id, 'hex'),           -- BYTEA to hex string\n            'span_id', encode(span_id, 'hex'),             -- BYTEA to hex string\n            'parent_span_id', encode(parent_span_id, 'hex'), -- BYTEA to hex string\n            'trace_state', trace_state,\n            'flags', flags,\n            'name', name,\n            'kind', kind,\n            'start_time_unix_nano', start_time_unix_nano,\n            'end_time_unix_nano', end_time_unix_nano,\n            'attributes', attributes,\n            'dropped_attributes_count', dropped_attributes_count,\n            'events', events,\n            'dropped_events_count', dropped_events_count,\n            'links', links,\n            'dropped_links_count', dropped_links_count,\n            'status', status\n        ) as \"span!\"\n        FROM otel_traces\n        WHERE trace_id = $1\n        ORDER BY start_time_unix_nano",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "span!",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bytea"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "90d93fd3bd91e468c1e796e41e31e4f15a825b442346c7386836127bc5723c93"
+}

backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881"
+}

backend/.sqlx/query-99398d6d6aa04235226f1a5d0f100aea034d7ee2c86aa8fa5ccec0e3560965fd.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT token_family FROM mcp_oauth_refresh_token\n                 WHERE refresh_token = $1 AND used_at IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token_family",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "99398d6d6aa04235226f1a5d0f100aea034d7ee2c86aa8fa5ccec0e3560965fd"
+}

backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81"
+}

backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8"
+}

backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f"
+}

backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "value",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018"
+}

backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.610.0"
+version = "1.613.4"
 authors.workspace = true
 edition.workspace = true
 
@@ -18,6 +18,7 @@ members = [
     "./windmill-indexer",
     "./windmill-macros",
     "./windmill-oauth",
+    "./windmill-local",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
     "./parsers/windmill-parser-go",
@@ -35,7 +36,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal"]
 
 [workspace.package]
-version = "1.610.0"
+version = "1.613.4"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -161,6 +162,7 @@ k8s-openapi.workspace = true
 libloading.workspace = true
 bitflags.workspace = true
 globset.workspace = true
+opentelemetry-proto.workspace = true
 
 
 [target.'cfg(windows)'.dependencies]
@@ -219,6 +221,8 @@ memchr = "2.7.4"
 axum = { version = "^0.7", features = ["multipart", "macros"] }
 headers = "^0"
 hyper = { version = "^1", features = ["full"] }
+hyper-tls = "^0.6"
+hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
 tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
 tokio-stream = { version = "0.1.17" }
 tower = "^0"
@@ -233,6 +237,7 @@ thiserror = "^2"
 anyhow = "^1"
 chrono = { version = "^0.4", features = ["serde"] }
 chrono-tz = "^0.10.1"
+derive_more = { version = "1", features = ["deref", "deref_mut"], default-features = false }
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 tracing-appender = "^0"
@@ -333,6 +338,7 @@ futures-core = "^0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
 const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] }
+const-str = "0.5"
 constant_time_eq = "0.3.1"
 dyn-iter = "0.2.0"
 rsa = "^0"
@@ -415,6 +421,7 @@ opentelemetry_sdk = { version = "0.27.1", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
 opentelemetry-appender-tracing = "0.27.0"
 opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
+opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] }
 
 bollard = "0.18.1"
 
@@ -456,3 +463,6 @@ oracle = { version = "0.6.3", features = ["chrono"] }
 rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
 strum = { version = "0.27", features = ["derive"] }
 strum_macros = "^0"
+hudsucker = { version = "0.22", features = ["rcgen-ca", "native-tls-client"] }
+hyper-http-proxy = { version = "1", default-features = false, features = ["native-tls"] }
+rcgen = "0.13"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-da1518ed54410478ca209f1d06298a1407be38a8
+0bfcfe263622f48c086331b628c1239d5cea0b37

backend/migrations/20260112000000_otel_traces.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS otel_traces_time_idx;
+DROP INDEX IF EXISTS otel_traces_trace_time_idx;
+DROP TABLE IF EXISTS otel_traces;

backend/migrations/20260112000000_otel_traces.up.sql

@@ -0,0 +1,36 @@
+-- OpenTelemetry Span storage (all fields from proto::Span).
+-- See: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto
+
+CREATE TABLE IF NOT EXISTS otel_traces (
+    -- Identity fields (BYTEA for efficient storage and querying)
+    trace_id             BYTEA NOT NULL,              -- 16 bytes (proto: bytes)
+    span_id              BYTEA NOT NULL,              -- 8 bytes (proto: bytes)
+    trace_state          TEXT NOT NULL DEFAULT '',    -- W3C trace-context (proto: string)
+    parent_span_id       BYTEA NOT NULL DEFAULT '', -- 8 bytes, empty if root span (proto: bytes)
+    flags                INTEGER NOT NULL DEFAULT 0, -- W3C trace flags (proto: fixed32)
+    -- Core fields
+    name                 TEXT NOT NULL,              -- operation name (proto: string)
+    kind                 INTEGER NOT NULL,           -- SpanKind enum (proto: int32)
+    start_time_unix_nano BIGINT NOT NULL,            -- (proto: fixed64, postgres has no u64)
+    end_time_unix_nano   BIGINT NOT NULL,            -- (proto: fixed64, postgres has no u64)
+    -- Attributes
+    attributes                 JSONB NOT NULL DEFAULT '[]', -- (proto: repeated KeyValue)
+    dropped_attributes_count   INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Events
+    events                     JSONB NOT NULL DEFAULT '[]', -- (proto: repeated Event)
+    dropped_events_count       INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Links
+    links                      JSONB NOT NULL DEFAULT '[]', -- (proto: repeated Link)
+    dropped_links_count        INTEGER NOT NULL DEFAULT 0,  -- (proto: uint32)
+    -- Status
+    status               JSONB,                      -- (proto: optional Status message)
+    PRIMARY KEY (trace_id, span_id)
+);
+
+-- Query spans by trace_id, ordered by time
+CREATE INDEX IF NOT EXISTS otel_traces_trace_time_idx ON otel_traces (trace_id, start_time_unix_nano);
+
+-- Time-based cleanup (retention policy)
+CREATE INDEX IF NOT EXISTS otel_traces_time_idx ON otel_traces (start_time_unix_nano);
+
+-- trace_id = job_id.as_bytes()

backend/migrations/20260120100000_mcp_oauth_server.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS mcp_oauth_refresh_token;
+DROP TABLE IF EXISTS mcp_oauth_server_code;
+DROP TABLE IF EXISTS mcp_oauth_server_client;

backend/migrations/20260120100000_mcp_oauth_server.up.sql

@@ -0,0 +1,44 @@
+-- OAuth server: clients that have registered with Windmill to access MCP
+-- Only public clients are supported (PKCE required, no client secrets)
+CREATE TABLE mcp_oauth_server_client (
+    client_id VARCHAR(255) PRIMARY KEY,
+    client_name VARCHAR(255) NOT NULL,
+    redirect_uris TEXT[] NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- OAuth server: authorization codes (short-lived, single-use)
+CREATE TABLE mcp_oauth_server_code (
+    code VARCHAR(64) PRIMARY KEY,
+    client_id VARCHAR(255) NOT NULL REFERENCES mcp_oauth_server_client(client_id) ON DELETE CASCADE,
+    user_email VARCHAR(255) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    scopes TEXT[] NOT NULL,
+    redirect_uri TEXT NOT NULL,
+    code_challenge VARCHAR(128),  -- PKCE
+    code_challenge_method VARCHAR(10),  -- 'S256' or 'plain'
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    expires_at TIMESTAMPTZ NOT NULL DEFAULT now() + INTERVAL '10 minutes'
+);
+
+CREATE INDEX idx_mcp_oauth_server_code_expires ON mcp_oauth_server_code(expires_at);
+
+-- MCP OAuth refresh tokens for token rotation
+CREATE TABLE mcp_oauth_refresh_token (
+    id BIGSERIAL PRIMARY KEY,
+    refresh_token VARCHAR(64) NOT NULL UNIQUE,
+    access_token VARCHAR(64) NOT NULL,
+    client_id VARCHAR(255) NOT NULL REFERENCES mcp_oauth_server_client(client_id) ON DELETE CASCADE,
+    user_email VARCHAR(255) NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    scopes TEXT[] NOT NULL,
+    token_family UUID NOT NULL,  -- Groups tokens from same auth flow for theft detection
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    expires_at TIMESTAMPTZ NOT NULL,
+    used_at TIMESTAMPTZ DEFAULT NULL,  -- For rotation tracking (single-use)
+    revoked BOOLEAN NOT NULL DEFAULT FALSE  -- For theft detection
+);
+
+CREATE INDEX idx_mcp_oauth_refresh_token_token ON mcp_oauth_refresh_token(refresh_token);
+CREATE INDEX idx_mcp_oauth_refresh_token_expires ON mcp_oauth_refresh_token(expires_at);
+CREATE INDEX idx_mcp_oauth_refresh_token_family ON mcp_oauth_refresh_token(token_family);

backend/migrations/20260120204751_custom_user_instance_createrole.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20260120204751_custom_user_instance_createrole.up.sql

@@ -0,0 +1,10 @@
+
+DO $$
+BEGIN
+  ALTER ROLE custom_instance_user CREATEROLE;
+EXCEPTION
+    WHEN others THEN
+        RAISE NOTICE 'Error in custom_instance_user migration: %', SQLERRM;
+        -- Continue without failing the migration
+END
+$$;

backend/parsers/windmill-parser-py-imports/src/mapping.rs

@@ -380,5 +380,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
     "taiga" => "python-taiga",
     "docx" => "python-docx",
     "vt" => "vt-py",
+    "grpc" => "grpcio",
     // Add new entry here ^
 };

backend/src/main.rs

@@ -44,7 +44,7 @@ use windmill_common::{
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
         HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
         INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
-        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, OTEL_TRACING_PROXY_SETTING,
         MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
         NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
         PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING,
@@ -99,9 +99,9 @@ use crate::monitor::{
     reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
     reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
     reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting,
-    reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting,
-    reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
-    reload_smtp_config, reload_worker_config, MonitorIteration,
+    reload_jwt_secret_setting, reload_license_key, reload_otel_tracing_proxy_setting,
+    reload_npm_config_registry_setting, reload_pip_index_url_setting, reload_retention_period_setting,
+    reload_scim_token_setting, reload_smtp_config, reload_worker_config, MonitorIteration,
 };
 
 #[cfg(feature = "parquet")]
@@ -452,6 +452,7 @@ async fn windmill_main() -> anyhow::Result<()> {
             .unwrap_or(DEFAULT_NUM_WORKERS as i32)
     };
 
+    // TODO: maybe gate behind debug_assertions?
     if num_workers > 1 && !std::env::var("WORKER_GROUP").is_ok_and(|x| x == "native") {
         println!(
             "We STRONGLY recommend using at most 1 worker per container, use at your own risks"
@@ -802,6 +803,10 @@ Windmill Community Edition {GIT_VERSION}
         #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
         let log_indexer_f = async { Ok(()) as anyhow::Result<()> };
 
+        // Resubscribe for OTEL tracing proxy before workers_f captures killpill_rx
+        #[cfg(all(feature = "private", feature = "enterprise"))]
+        let otel_killpill_rx = killpill_rx.resubscribe();
+
         let server_f = async {
             if !is_agent {
                 if let Some(db) = conn.as_sql() {
@@ -1156,6 +1161,13 @@ Windmill Community Edition {GIT_VERSION}
                                                         KEEP_JOB_DIR_SETTING => {
                                                             load_keep_job_dir(&conn).await;
                                                         },
+                                                        OTEL_TRACING_PROXY_SETTING => {
+                                                            reload_otel_tracing_proxy_setting(&conn).await;
+                                                            if worker_mode {
+                                                                tracing::info!("OTEL tracing proxy setting changed, restarting worker");
+                                                                send_delayed_killpill(&tx, 4, "OTEL tracing proxy setting change").await;
+                                                            }
+                                                        },
                                                         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
                                                             load_require_preexisting_user(&db).await;
                                                         },
@@ -1329,7 +1341,8 @@ Windmill Community Edition {GIT_VERSION}
 
                                 // update min version explicitly.
                                 // for sql connection it is the part of monitor_db.
-                                windmill_common::worker::update_min_version(conn).await;
+                                // TODO: pass worker names for min keep-alive alerts (for HTTP connection)
+                                windmill_common::min_version::update_min_version(conn, false, vec![], false).await;
                             }
                         };
                     }
@@ -1364,6 +1377,42 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
+        let otel_tracing_proxy_f = async {
+            #[cfg(all(feature = "private", feature = "enterprise"))]
+            {
+                // Start OTEL tracing proxy for HTTP request interception
+                // Only enabled when: setting is on, worker mode (not server), and single worker (to avoid race conditions)
+                if worker_mode
+                    && num_workers == 1
+                    && windmill_worker::OTEL_TRACING_PROXY_SETTINGS
+                        .read()
+                        .await
+                        .enabled
+                {
+                    if let Some(db) = conn.as_sql() {
+                        tracing::info!(
+                            "Starting OTEL tracing proxy (port will be dynamically assigned)"
+                        );
+                        if let Err(e) =
+                            windmill_worker::start_otel_tracing_proxy(db.clone(), otel_killpill_rx)
+                                .await
+                        {
+                            tracing::error!("OTEL tracing proxy error: {}", e);
+                        }
+                    }
+                } else if windmill_worker::OTEL_TRACING_PROXY_SETTINGS
+                    .read()
+                    .await
+                    .enabled
+                    && num_workers > 1
+                {
+                    tracing::warn!("OTEL tracing proxy is enabled but num_workers > 1. Disabling to avoid race conditions. Set NUM_WORKERS=1 to enable.");
+                }
+            }
+
+            Ok(()) as anyhow::Result<()>
+        };
+
         if server_mode {
             if let Some(db) = conn.as_sql() {
                 schedule_stats(&db, &HTTP_CLIENT).await;
@@ -1378,6 +1427,7 @@ Windmill Community Edition {GIT_VERSION}
                 monitor_f,
                 server_f,
                 metrics_f,
+                otel_tracing_proxy_f,
                 indexer_f,
                 log_indexer_f
             )?;

backend/src/monitor.rs

@@ -53,7 +53,7 @@ use windmill_common::{
         HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
         JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
         LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
+        OTEL_TRACING_PROXY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
         POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
@@ -81,10 +81,10 @@ use windmill_common::{
 use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING};
 use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
 use windmill_worker::{
-    handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
-    INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN,
-    NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT,
-    POWERSHELL_REPO_URL,
+    handle_job_error, JobCompletedSender, OtelTracingProxySettings, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
+    INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS,
+    OTEL_TRACING_PROXY_SETTINGS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL,
+    PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL,
 };
 
 #[cfg(feature = "parquet")]
@@ -320,6 +320,7 @@ pub async fn initial_load(
         reload_maven_repos_setting(&conn).await;
         reload_no_default_maven_setting(&conn).await;
         reload_ruby_repos_setting(&conn).await;
+        reload_otel_tracing_proxy_setting(&conn).await;
     }
 }
 
@@ -778,6 +779,35 @@ pub async fn load_keep_job_dir(conn: &Connection) {
     };
 }
 
+pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) {
+    match load_value_from_global_settings_with_conn(conn, OTEL_TRACING_PROXY_SETTING, true).await {
+        Ok(Some(settings)) => {
+            match serde_json::from_value::<OtelTracingProxySettings>(settings) {
+                Ok(new_settings) => {
+                    let mut current = OTEL_TRACING_PROXY_SETTINGS.write().await;
+                    if current.enabled != new_settings.enabled
+                        || current.enabled_languages != new_settings.enabled_languages
+                    {
+                        tracing::info!(
+                            "OTEL tracing proxy settings changed: enabled={}, languages={:?}",
+                            new_settings.enabled,
+                            new_settings.enabled_languages
+                        );
+                        *current = new_settings;
+                    }
+                }
+                Err(e) => {
+                    tracing::error!("Error parsing OTEL tracing proxy settings: {e:#}");
+                }
+            }
+        }
+        Err(e) => {
+            tracing::error!("Error loading OTEL tracing proxy setting: {e:#}");
+        }
+        _ => (),
+    };
+}
+
 pub async fn load_require_preexisting_user(db: &DB) {
     let value =
         load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await;
@@ -829,6 +859,22 @@ pub async fn delete_expired_items(db: &DB) -> () {
         Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
     }
 
+    // Clean up expired MCP OAuth refresh tokens
+    let mcp_refresh_tokens_r: std::result::Result<Vec<i64>, _> = sqlx::query_scalar(
+        "DELETE FROM mcp_oauth_refresh_token WHERE expires_at <= now() RETURNING id",
+    )
+    .fetch_all(db)
+    .await;
+
+    match mcp_refresh_tokens_r {
+        Ok(ids) => {
+            if ids.len() > 0 {
+                tracing::info!("deleted {} expired MCP OAuth refresh tokens", ids.len())
+            }
+        }
+        Err(e) => tracing::error!("Error deleting MCP OAuth refresh tokens: {}", e.to_string()),
+    }
+
     let deleted_cache = sqlx::query_scalar!(
             "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
         )
@@ -922,6 +968,23 @@ pub async fn delete_expired_items(db: &DB) -> () {
         Err(e) => tracing::error!("Error deleting expired blacklisted agent tokens: {:?}", e),
     }
 
+    match sqlx::query_scalar!(
+        "DELETE FROM mcp_oauth_server_code WHERE expires_at <= now() RETURNING code",
+    )
+    .fetch_all(db)
+    .await
+    {
+        Ok(deleted_codes) => {
+            if deleted_codes.len() > 0 {
+                tracing::info!(
+                    "deleted {} expired MCP OAuth authorization codes",
+                    deleted_codes.len()
+                );
+            }
+        }
+        Err(e) => tracing::error!("Error deleting expired MCP OAuth authorization codes: {:?}", e),
+    }
+
     let job_retention_secs = *JOB_RETENTION_SECS.read().await;
     if job_retention_secs > 0 {
         let batch_size = *JOB_CLEANUP_BATCH_SIZE;
@@ -1800,7 +1863,7 @@ pub async fn monitor_db(
 
     let update_min_worker_version_f = async {
         #[cfg(not(feature = "test_job_debouncing"))]
-        windmill_common::worker::update_min_version(conn).await;
+        windmill_common::min_version::update_min_version(conn, _worker_mode, WORKERS_NAMES.read().await.clone(), initial_load).await;
     };
 
     join!(
@@ -2888,8 +2951,8 @@ RETURNING key,job_id
 
 async fn cleanup_debounce_keys_for_completed_jobs(db: &DB) -> error::Result<()> {
     // If min version doesn't support runnable settings, clean up debounce keys for completed jobs
-    if !*windmill_common::worker::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
-        .read()
+    if !windmill_common::min_version::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
+        .met()
         .await
     {
         let result = sqlx::query!(

backend/tests/common/mod.rs

@@ -1,12 +1,11 @@
 #![allow(dead_code)]
 
-use std::{future::Future, str::FromStr, sync::Arc};
+use std::{future::Future, str::FromStr};
 
 use futures::Stream;
 use serde::Serialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, Pool, Postgres};
-use tokio::sync::RwLock;
 use uuid::Uuid;
 use windmill_api_client::types::NewScript;
 #[cfg(feature = "python")]
@@ -498,11 +497,11 @@ pub async fn initialize_tracing() {
 }
 
 pub async fn test_for_versions<F: Future<Output = ()>>(
-    version_flags: impl Iterator<Item = Arc<RwLock<bool>>>,
+    constraints: impl Iterator<Item = &'static windmill_common::min_version::VersionConstraint>,
     test: impl Fn() -> F,
 ) {
-    for version_flag in version_flags {
-        *version_flag.write().await = true;
+    for constraint in constraints {
+        *windmill_common::min_version::MIN_VERSION.write().await = constraint.version().clone();
         test().await;
     }
 }

backend/tests/fixtures/secret_backend.sql

@@ -0,0 +1,31 @@
+-- Fixture for secret backend migration tests
+-- Sets up test secrets in the variable table
+
+-- Create a second workspace for testing workspace isolation
+INSERT INTO workspace (id, name, owner)
+VALUES ('test-workspace-2', 'test-workspace-2', 'test-user')
+ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_settings (workspace_id)
+VALUES ('test-workspace-2')
+ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_key(workspace_id, kind, key)
+VALUES ('test-workspace-2', 'cloud', 'test-key-2')
+ON CONFLICT DO NOTHING;
+
+-- Insert test secrets for workspace 1
+-- Note: The 'value' column stores encrypted values in production,
+-- but for tests we'll use plain text that the migration will handle
+INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
+VALUES
+    ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'),
+    ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'),
+    ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}')
+ON CONFLICT DO NOTHING;
+
+-- Insert test secrets for workspace 2 (to test isolation)
+INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
+VALUES
+    ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}')
+ON CONFLICT DO NOTHING;

backend/tests/job_payload.rs

@@ -3,16 +3,15 @@ mod common;
 mod job_payload {
     use serde_json::json;
     use sqlx::{Pool, Postgres};
-    use std::sync::Arc;
-    use tokio::sync::RwLock;
-    use windmill_common::flow_status::RestartedFrom;
     use windmill_common::flows::{FlowModule, FlowModuleValue, FlowValue};
     use windmill_common::jobs::JobPayload;
     use windmill_common::scripts::{ScriptHash, ScriptLang};
+    use windmill_common::flow_status::RestartedFrom;
 
     use crate::common::*;
-    use windmill_common::worker::{
-        MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440,
+    use windmill_common::min_version::{
+        MIN_VERSION, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432,
+        MIN_VERSION_IS_AT_LEAST_1_440,
     };
 
     pub async fn initialize_tracing() {
@@ -28,18 +27,15 @@ mod job_payload {
         });
     }
 
-    use lazy_static::lazy_static;
-
     use windmill_common::cache;
     use windmill_common::flows::FlowNodeId;
+    use windmill_common::min_version::VersionConstraint;
 
-    lazy_static! {
-        static ref VERSION_FLAGS: [Arc<RwLock<bool>>; 3] = [
-            MIN_VERSION_IS_AT_LEAST_1_427.clone(),
-            MIN_VERSION_IS_AT_LEAST_1_432.clone(),
-            MIN_VERSION_IS_AT_LEAST_1_440.clone(),
-        ];
-    }
+    const VERSION_FLAGS: [&VersionConstraint; 3] = [
+        &MIN_VERSION_IS_AT_LEAST_1_427,
+        &MIN_VERSION_IS_AT_LEAST_1_432,
+        &MIN_VERSION_IS_AT_LEAST_1_440,
+    ];
 
     #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
@@ -71,7 +67,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello foo!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -116,7 +112,7 @@ mod job_payload {
                 .unwrap();
             assert_eq!(job.preprocessed, Some(true));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -183,7 +179,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello foo!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         let test = || async {
             let result = RunJob::from(JobPayload::FlowScript {
                 id: flow_scripts[1],
@@ -206,7 +202,7 @@ mod job_payload {
                 json!("Did you just say \"You know nothing Jean Neige\"??!")
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -252,7 +248,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Did you just say \"Hello tests!\"??!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -282,19 +278,19 @@ mod job_payload {
 
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_427(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_427.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_427.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_432(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_432.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_432.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_dependencies_payload_min_1_440(db: Pool<Postgres>) -> anyhow::Result<()> {
-        *MIN_VERSION_IS_AT_LEAST_1_440.write().await = true;
+        *MIN_VERSION.write().await = MIN_VERSION_IS_AT_LEAST_1_440.version().clone();
         test_dependencies_payload(db).await?;
         Ok(())
     }
@@ -323,7 +319,7 @@ mod job_payload {
                 &json!("Successful lock file generation")
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -383,7 +379,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello Jean Neige!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -414,7 +410,7 @@ mod job_payload {
                 json!({ "lock": "", "status": "Successful lock file generation" })
             );
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -447,7 +443,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_nodes_flow".to_string(),
@@ -460,7 +456,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -520,7 +516,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_preprocessor".to_string(),
@@ -533,7 +529,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -577,7 +573,7 @@ mod job_payload {
             );
         };
         // Test the not "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         // Deploy the flow to produce the "lite" version.
         let _ = RunJob::from(JobPayload::FlowDependencies {
             path: "f/system/hello_with_nodes_flow".to_string(),
@@ -590,7 +586,7 @@ mod job_payload {
         .json_result()
         .unwrap();
         // Test the "lite" flow.
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 
@@ -638,7 +634,7 @@ mod job_payload {
 
             assert_eq!(result, json!("Hello Jean Neige!"));
         };
-        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
         Ok(())
     }
 

backend/tests/list_jobs.rs

@@ -0,0 +1,900 @@
+use serde::Deserialize;
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+
+use windmill_common::{
+    jobs::{JobPayload, RawCode},
+    scripts::ScriptLang,
+};
+
+mod common;
+use common::*;
+
+#[derive(Debug, Deserialize)]
+struct ListJobsResponse {
+    #[serde(rename = "type")]
+    #[allow(dead_code)]
+    typ: String,
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Response struct for queue/list endpoint (no type field)
+#[derive(Debug, Deserialize)]
+struct QueueJobResponse {
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Response struct for completed/list endpoint
+#[cfg(feature = "python")]
+#[derive(Debug, Deserialize)]
+struct CompletedJobResponse {
+    id: String,
+    #[serde(default)]
+    args: Option<serde_json::Value>,
+    #[serde(flatten)]
+    _extra: std::collections::HashMap<String, serde_json::Value>,
+}
+
+/// Test that list_jobs returns jobs without args by default
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call list_jobs without include_args
+    let response = client
+        .client()
+        .get(format!("{}/w/test-workspace/jobs/list", client.baseurl()))
+        .send()
+        .await?;
+
+    assert!(response.status().is_success(), "list_jobs should succeed");
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns jobs with args when include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns completed jobs with args when include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_completed_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included for completed jobs when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value for completed jobs"
+    );
+
+    Ok(())
+}
+
+/// Test that list_jobs returns both queued and completed jobs with args
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_mixed_queue_and_completed(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion first
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(completed_arg): return completed_arg".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("completed_arg", json!("completed_value"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let completed_job_id = completed_job.id;
+
+    // Push another job to the queue (don't run it)
+    let queued_job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(queued_arg): return queued_arg".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("queued_arg", json!("queued_value"))
+    .push(&db)
+    .await;
+
+    // Call list_jobs with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "list_jobs with include_args should succeed"
+    );
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find the completed job
+    let completed = jobs
+        .iter()
+        .find(|j| j.id == completed_job_id.to_string())
+        .expect("should find the completed job");
+
+    assert!(
+        completed.args.is_some(),
+        "completed job should have args when include_args=true"
+    );
+    assert_eq!(
+        completed.args.as_ref().unwrap().get("completed_arg"),
+        Some(&json!("completed_value")),
+        "completed job should have correct args"
+    );
+
+    // Find the queued job
+    let queued = jobs
+        .iter()
+        .find(|j| j.id == queued_job_id.to_string())
+        .expect("should find the queued job");
+
+    assert!(
+        queued.args.is_some(),
+        "queued job should have args when include_args=true"
+    );
+    assert_eq!(
+        queued.args.as_ref().unwrap().get("queued_arg"),
+        Some(&json!("queued_value")),
+        "queued job should have correct args"
+    );
+
+    Ok(())
+}
+
+/// Test list_jobs with multiple queued jobs and include_args
+#[sqlx::test(fixtures("base"))]
+async fn test_list_jobs_multiple_queued_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push two jobs with different args
+    let job1_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(1))
+    .push(&db)
+    .await;
+
+    let job2_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(y): return y".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("y", json!(2))
+    .push(&db)
+    .await;
+
+    // Test with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(response.status().is_success(), "list_jobs should succeed");
+
+    let jobs: Vec<ListJobsResponse> = response.json().await?;
+
+    // Find both jobs
+    let job1 = jobs.iter().find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs.iter().find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("x"),
+        Some(&json!(1)),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("y"),
+        Some(&json!(2)),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}
+
+// ============================================================================
+// Tests for /queue/list endpoint
+// ============================================================================
+
+/// Test that queue/list returns jobs without args by default
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call queue/list without include_args
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that queue/list returns jobs with args when include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push a job to the queue with specific args
+    let job_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .push(&db)
+    .await;
+
+    // Call queue/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list with include_args should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find the job we created
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the created job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test queue/list with multiple jobs and include_args=true
+#[sqlx::test(fixtures("base"))]
+async fn test_queue_list_multiple_jobs_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Push two jobs with different args
+    let job1_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(a): return a".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("a", json!("value_a"))
+    .push(&db)
+    .await;
+
+    let job2_id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(b): return b".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("b", json!("value_b"))
+    .push(&db)
+    .await;
+
+    // Call queue/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/queue/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "queue/list should succeed"
+    );
+
+    let jobs: Vec<QueueJobResponse> = response.json().await?;
+
+    // Find both jobs
+    let job1 = jobs
+        .iter()
+        .find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs
+        .iter()
+        .find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("a"),
+        Some(&json!("value_a")),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("b"),
+        Some(&json!("value_b")),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}
+
+// ============================================================================
+// Tests for /completed/list endpoint
+// ============================================================================
+
+/// Test that completed/list returns jobs without args by default
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_without_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call completed/list without include_args
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be None when include_args is not set
+    assert!(
+        job.args.is_none(),
+        "args should not be included when include_args is not set"
+    );
+
+    Ok(())
+}
+
+/// Test that completed/list returns jobs with args when include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_with_include_args(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run a job to completion
+    let completed_job = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(x): return x * 2".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("x", json!(42))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job_id = completed_job.id;
+
+    // Call completed/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list with include_args should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find the completed job
+    let job = jobs
+        .iter()
+        .find(|j| j.id == job_id.to_string())
+        .expect("should find the completed job");
+
+    // Args should be present when include_args=true
+    assert!(
+        job.args.is_some(),
+        "args should be included when include_args=true"
+    );
+
+    let args = job.args.as_ref().unwrap();
+    assert_eq!(
+        args.get("x"),
+        Some(&json!(42)),
+        "args should contain the correct value"
+    );
+
+    Ok(())
+}
+
+/// Test completed/list with multiple jobs and include_args=true
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base"))]
+async fn test_completed_list_multiple_jobs_with_include_args(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    // Run first job to completion
+    let completed_job1 = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(a): return a".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("a", json!("completed_a"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job1_id = completed_job1.id;
+
+    // Run second job to completion
+    let completed_job2 = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "def main(b): return b".to_string(),
+        path: None,
+        language: ScriptLang::Python3,
+        lock: None,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings:
+            windmill_common::runnable_settings::ConcurrencySettings::default().into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+    }))
+    .arg("b", json!("completed_b"))
+    .run_until_complete(&db, false, port)
+    .await;
+
+    let job2_id = completed_job2.id;
+
+    // Call completed/list with include_args=true
+    let response = client
+        .client()
+        .get(format!(
+            "{}/w/test-workspace/jobs/completed/list?include_args=true",
+            client.baseurl()
+        ))
+        .send()
+        .await?;
+
+    assert!(
+        response.status().is_success(),
+        "completed/list should succeed"
+    );
+
+    let jobs: Vec<CompletedJobResponse> = response.json().await?;
+
+    // Find both completed jobs
+    let job1 = jobs
+        .iter()
+        .find(|j| j.id == job1_id.to_string())
+        .expect("should find job1");
+    let job2 = jobs
+        .iter()
+        .find(|j| j.id == job2_id.to_string())
+        .expect("should find job2");
+
+    // Both should have args
+    assert!(job1.args.is_some(), "job1 args should be included");
+    assert!(job2.args.is_some(), "job2 args should be included");
+
+    // Check the args are correct
+    assert_eq!(
+        job1.args.as_ref().unwrap().get("a"),
+        Some(&json!("completed_a")),
+        "job1 should have correct args"
+    );
+    assert_eq!(
+        job2.args.as_ref().unwrap().get("b"),
+        Some(&json!("completed_b")),
+        "job2 should have correct args"
+    );
+
+    Ok(())
+}

backend/tests/secret_backend_integration.rs

@@ -0,0 +1,557 @@
+//! Integration tests for HashiCorp Vault secret backend.
+//!
+//! These tests require:
+//! 1. A PostgreSQL database (handled by sqlx test framework)
+//! 2. A running HashiCorp Vault instance
+//! 3. The RUN_VAULT_TESTS=1 environment variable to be set
+//!
+//! Environment variables:
+//! - RUN_VAULT_TESTS=1  - Required to run the tests
+//! - VAULT_ADDR         - Vault server address (default: http://127.0.0.1:8200)
+//! - VAULT_TOKEN        - Static token for static token tests (default: test-root-token)
+//! - BASE_URL           - Windmill instance URL for JWT tests (default: http://localhost:8000)
+//!
+//! Run tests (static token mode):
+//! ```bash
+//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \
+//!     secret_backend_integration --features private,enterprise -- --nocapture
+//! ```
+//!
+//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint):
+//! ```bash
+//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \
+//!     secret_backend_integration --features private,enterprise,openidconnect -- --nocapture
+//! ```
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+mod tests {
+    use sqlx::{Pool, Postgres};
+    use std::collections::HashMap;
+    use windmill_common::secret_backend::{
+        migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection,
+        SecretBackend, VaultBackend, VaultSettings,
+    };
+
+    /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var)
+    fn should_run_vault_tests() -> bool {
+        std::env::var("RUN_VAULT_TESTS")
+            .map(|v| v == "1" || v.to_lowercase() == "true")
+            .unwrap_or(false)
+    }
+
+    /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation)
+    async fn setup_base_url() {
+        let base_url = std::env::var("BASE_URL")
+            .unwrap_or_else(|_| "http://localhost:8000".to_string());
+        let mut url = windmill_common::BASE_URL.write().await;
+        *url = base_url;
+    }
+
+    /// Skip test if RUN_VAULT_TESTS is not set
+    macro_rules! skip_if_no_vault {
+        () => {
+            if !should_run_vault_tests() {
+                println!("Skipping test: RUN_VAULT_TESTS=1 not set");
+                println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ...");
+                return;
+            }
+        };
+    }
+
+    fn vault_settings_static_token() -> VaultSettings {
+        VaultSettings {
+            address: std::env::var("VAULT_ADDR")
+                .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+            mount_path: "windmill".to_string(),
+            jwt_role: None, // Static token mode
+            namespace: None,
+            token: Some(
+                std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
+            ),
+        }
+    }
+
+    fn vault_settings_jwt() -> VaultSettings {
+        VaultSettings {
+            address: std::env::var("VAULT_ADDR")
+                .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+            mount_path: "windmill".to_string(),
+            jwt_role: Some("windmill-secrets".to_string()), // JWT mode
+            namespace: None,
+            token: None, // No static token - use JWT
+        }
+    }
+
+    // ==================== Static Token Tests ====================
+
+    /// Test Vault connection with static token
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_connection_static_token(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        println!("Testing Vault connection with static token...");
+        println!("  Address: {}", settings.address);
+
+        let result = test_vault_connection(&settings, Some(&db)).await;
+        assert!(
+            result.is_ok(),
+            "Failed to connect to Vault: {:?}",
+            result.err()
+        );
+        println!("✓ Successfully connected to Vault with static token");
+    }
+
+    /// Test basic CRUD operations with static token
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_crud_static_token(_db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-crud-static";
+        let path = "test-secret";
+        let value = "my-super-secret-value-123";
+
+        println!("Testing CRUD with static token...");
+
+        // Create
+        println!("  Creating secret...");
+        backend
+            .set_secret(workspace_id, path, value)
+            .await
+            .expect("Failed to create secret");
+        println!("  ✓ Created");
+
+        // Read
+        println!("  Reading secret...");
+        let read_value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read secret");
+        assert_eq!(read_value, value);
+        println!("  ✓ Read (value matches)");
+
+        // Update
+        println!("  Updating secret...");
+        let new_value = "updated-secret-value-456";
+        backend
+            .set_secret(workspace_id, path, new_value)
+            .await
+            .expect("Failed to update secret");
+        let updated = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read updated secret");
+        assert_eq!(updated, new_value);
+        println!("  ✓ Updated");
+
+        // Delete
+        println!("  Deleting secret...");
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete secret");
+        let result = backend.get_secret(workspace_id, path).await;
+        assert!(result.is_err(), "Secret should be deleted");
+        println!("  ✓ Deleted");
+
+        println!("✓ CRUD operations successful with static token");
+    }
+
+    // ==================== JWT Auth Tests ====================
+
+    /// Test Vault connection with JWT authentication
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_connection_jwt(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        setup_base_url().await;
+
+        let settings = vault_settings_jwt();
+        println!("Testing Vault connection with JWT auth...");
+        println!("  Address: {}", settings.address);
+        println!("  JWT Role: {:?}", settings.jwt_role);
+        println!("  BASE_URL: {}", windmill_common::BASE_URL.read().await.clone());
+
+        let result = test_vault_connection(&settings, Some(&db)).await;
+        assert!(
+            result.is_ok(),
+            "Failed to connect to Vault with JWT: {:?}",
+            result.err()
+        );
+        println!("✓ Successfully connected to Vault with JWT auth");
+    }
+
+    /// Test basic CRUD operations with JWT authentication
+    #[cfg(feature = "openidconnect")]
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_vault_crud_jwt(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        setup_base_url().await;
+
+        let settings = vault_settings_jwt();
+        let backend = VaultBackend::new_with_db(settings, db.clone());
+
+        let workspace_id = "test-crud-jwt";
+        let path = "jwt-test-secret";
+        let value = "jwt-authenticated-secret-value";
+
+        println!("Testing CRUD with JWT auth...");
+
+        // Create
+        println!("  Creating secret...");
+        backend
+            .set_secret(workspace_id, path, value)
+            .await
+            .expect("Failed to create secret with JWT");
+        println!("  ✓ Created");
+
+        // Read
+        println!("  Reading secret...");
+        let read_value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read secret with JWT");
+        assert_eq!(read_value, value);
+        println!("  ✓ Read (value matches)");
+
+        // Delete (cleanup)
+        println!("  Deleting secret...");
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete secret with JWT");
+        println!("  ✓ Deleted");
+
+        println!("✓ CRUD operations successful with JWT auth");
+    }
+
+    // ==================== Migration Tests ====================
+
+    /// Test migration from database to Vault
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        // Verify Vault connection
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // Check initial state
+        let secrets_before = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query secrets");
+
+        println!(
+            "Found {} secrets in database before migration:",
+            secrets_before.len()
+        );
+        for s in &secrets_before {
+            println!("  - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+        }
+
+        // Run migration
+        println!("\nMigrating secrets to Vault...");
+        let report = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration to Vault failed");
+
+        println!("Migration report:");
+        println!("  Total secrets: {}", report.total_secrets);
+        println!("  Migrated: {}", report.migrated_count);
+        println!("  Failed: {}", report.failed_count);
+
+        if !report.failures.is_empty() {
+            println!("  Failures:");
+            for f in &report.failures {
+                println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+            }
+        }
+
+        assert_eq!(report.failed_count, 0, "Migration had failures");
+        assert!(report.migrated_count > 0, "No secrets were migrated");
+
+        // Verify secrets in Vault
+        println!("\nVerifying secrets in Vault...");
+        let vault_backend = VaultBackend::new(settings.clone());
+
+        for secret in &secrets_before {
+            let result = vault_backend
+                .get_secret(&secret.workspace_id, &secret.path)
+                .await;
+            assert!(
+                result.is_ok(),
+                "Failed to read secret {}/{} from Vault: {:?}",
+                secret.workspace_id,
+                secret.path,
+                result.err()
+            );
+            println!(
+                "  ✓ {}/{} exists in Vault",
+                secret.workspace_id, secret.path
+            );
+        }
+
+        println!("\n✓ Migration to Vault completed successfully");
+    }
+
+    /// Test migration from Vault back to database
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // First migrate TO Vault
+        println!("Setting up: migrating secrets to Vault first...");
+        let to_vault = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Initial migration to Vault failed");
+        assert!(to_vault.migrated_count > 0, "No secrets to test with");
+        println!("  Migrated {} secrets to Vault", to_vault.migrated_count);
+
+        // Clear database values
+        println!("\nClearing database secret values...");
+        sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
+            .execute(&db)
+            .await
+            .expect("Failed to clear values");
+
+        // Migrate back from Vault
+        println!("\nMigrating secrets from Vault to database...");
+        let report = migrate_secrets_to_database(&db, &settings)
+            .await
+            .expect("Migration to database failed");
+
+        println!("Migration report:");
+        println!("  Total secrets: {}", report.total_secrets);
+        println!("  Migrated: {}", report.migrated_count);
+        println!("  Failed: {}", report.failed_count);
+
+        assert_eq!(report.failed_count, 0, "Migration had failures");
+        assert!(report.migrated_count > 0, "No secrets were migrated");
+
+        // Verify restored
+        let restored = sqlx::query!(
+            "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'"
+        )
+        .fetch_one(&db)
+        .await
+        .expect("Failed to count restored");
+
+        assert!(
+            restored.count.unwrap_or(0) > 0,
+            "No secrets were restored in database"
+        );
+
+        println!("\n✓ Migration to database completed successfully");
+    }
+
+    // ==================== Variable Rename Tests ====================
+
+    /// Test renaming a variable path in Vault
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_variable_rename(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+        let _ = &db; // suppress unused warning
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-workspace";
+        let old_path = "u/test-user/old_secret_name";
+        let new_path = "u/test-user/new_secret_name";
+        let value = "secret-value-for-rename-test";
+
+        println!("Testing variable rename in Vault...");
+
+        // Create secret at old path
+        println!("  Creating secret at old path: {}", old_path);
+        backend
+            .set_secret(workspace_id, old_path, value)
+            .await
+            .expect("Failed to create secret");
+
+        // Verify it exists
+        let read_value = backend
+            .get_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to read secret at old path");
+        assert_eq!(read_value, value);
+        println!("  ✓ Secret exists at old path");
+
+        // Simulate rename: read from old, write to new, delete old
+        println!("  Renaming: {} -> {}", old_path, new_path);
+        let secret_value = backend
+            .get_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to read for rename");
+
+        backend
+            .set_secret(workspace_id, new_path, &secret_value)
+            .await
+            .expect("Failed to write to new path");
+
+        backend
+            .delete_secret(workspace_id, old_path)
+            .await
+            .expect("Failed to delete old path");
+
+        // Verify old path is gone
+        let old_result = backend.get_secret(workspace_id, old_path).await;
+        assert!(old_result.is_err(), "Old path should not exist");
+        println!("  ✓ Old path deleted");
+
+        // Verify new path exists with correct value
+        let new_value = backend
+            .get_secret(workspace_id, new_path)
+            .await
+            .expect("Failed to read new path");
+        assert_eq!(new_value, value);
+        println!("  ✓ New path exists with correct value");
+
+        // Cleanup
+        backend
+            .delete_secret(workspace_id, new_path)
+            .await
+            .expect("Failed to cleanup");
+
+        println!("\n✓ Variable rename completed successfully");
+    }
+
+    // ==================== Full Round Trip Test ====================
+
+    /// Test full round-trip: DB -> Vault -> DB with verification
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_full_round_trip(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+
+        test_vault_connection(&settings, Some(&db))
+            .await
+            .expect("Failed to connect to Vault");
+
+        // Get original secrets
+        let original: HashMap<(String, String), String> = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query")
+        .into_iter()
+        .map(|r| ((r.workspace_id, r.path), r.value))
+        .collect();
+
+        println!("Original secrets: {} entries", original.len());
+
+        // Step 1: DB -> Vault
+        println!("\n=== Step 1: Migrate DB -> Vault ===");
+        let to_vault = migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration to Vault failed");
+        println!("Migrated {} secrets to Vault", to_vault.migrated_count);
+        assert_eq!(to_vault.failed_count, 0);
+
+        // Step 2: Clear DB
+        println!("\n=== Step 2: Clear database values ===");
+        sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
+            .execute(&db)
+            .await
+            .expect("Failed to clear");
+
+        // Step 3: Vault -> DB
+        println!("\n=== Step 3: Migrate Vault -> DB ===");
+        let to_db = migrate_secrets_to_database(&db, &settings)
+            .await
+            .expect("Migration to database failed");
+        println!("Migrated {} secrets to database", to_db.migrated_count);
+        assert_eq!(to_db.failed_count, 0);
+
+        // Step 4: Verify
+        println!("\n=== Step 4: Verify round-trip integrity ===");
+        let restored: HashMap<(String, String), String> = sqlx::query!(
+            "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+        )
+        .fetch_all(&db)
+        .await
+        .expect("Failed to query")
+        .into_iter()
+        .map(|r| ((r.workspace_id, r.path), r.value))
+        .collect();
+
+        for ((ws, path), _) in &original {
+            let restored_value = restored
+                .get(&(ws.clone(), path.clone()))
+                .expect(&format!("Secret {}/{} not found after round-trip", ws, path));
+
+            assert_ne!(
+                restored_value, "ROUND_TRIP_CLEARED",
+                "Secret {}/{} was not restored",
+                ws, path
+            );
+            println!("  ✓ {}/{}: restored", ws, path);
+        }
+
+        println!("\n✓ Full round-trip completed successfully!");
+    }
+
+    // ==================== Workspace Isolation Test ====================
+
+    /// Test that workspace isolation is maintained
+    #[sqlx::test(fixtures("base", "secret_backend"))]
+    async fn test_workspace_isolation(db: Pool<Postgres>) {
+        skip_if_no_vault!();
+
+        let settings = vault_settings_static_token();
+        let backend = VaultBackend::new(settings.clone());
+
+        // First migrate secrets to Vault
+        migrate_secrets_to_vault(&db, &settings)
+            .await
+            .expect("Migration failed");
+
+        println!("Testing workspace isolation...");
+
+        // Try to access workspace-2 secret from workspace-1 path (should fail)
+        let cross_access = backend
+            .get_secret("test-workspace", "u/test-user/other_secret")
+            .await;
+
+        assert!(
+            cross_access.is_err(),
+            "Cross-workspace access should fail!"
+        );
+        println!("✓ Cross-workspace access correctly denied");
+
+        // Verify own workspace access works
+        let ws1 = backend
+            .get_secret("test-workspace", "u/test-user/db_password")
+            .await;
+        assert!(ws1.is_ok(), "Same-workspace access should work");
+        println!("✓ Same-workspace access works");
+
+        println!("\n✓ Workspace isolation verified!");
+    }
+}
+
+// OSS version - just a placeholder to avoid compilation errors
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+mod tests {
+    #[test]
+    fn test_vault_requires_enterprise() {
+        println!("Vault integration tests require Enterprise Edition features");
+        println!("Run with: cargo test --features private,enterprise,openidconnect");
+    }
+}

backend/tests/secret_backend_migration.rs

@@ -0,0 +1,323 @@
+//! Integration tests for secret backend migration between database and HashiCorp Vault.
+//!
+//! These tests require:
+//! 1. A PostgreSQL database (handled by sqlx test framework)
+//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200
+//!
+//! To run these tests:
+//! ```bash
+//! # Start Vault in dev mode
+//! podman run -d --name vault-test -p 8200:8200 \
+//!     -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \
+//!     docker.io/hashicorp/vault:latest
+//!
+//! # Enable KV v2 secrets engine
+//! curl -s -H "X-Vault-Token: test-root-token" -X POST \
+//!     --data '{"type":"kv-v2"}' \
+//!     http://127.0.0.1:8200/v1/sys/mounts/windmill
+//!
+//! # Run the tests
+//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture
+//! ```
+
+use sqlx::{Pool, Postgres};
+use windmill_common::error::Result;
+use windmill_common::secret_backend::{
+    vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend},
+    SecretBackend, VaultSettings,
+};
+
+mod common;
+
+fn test_vault_settings() -> VaultSettings {
+    VaultSettings {
+        address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
+        mount_path: "windmill".to_string(),
+        jwt_role: Some("windmill-secrets".to_string()),
+        namespace: None,
+        token: Some(
+            std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()),
+        ),
+    }
+}
+
+/// Test that we can connect to Vault
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_vault_connection_works(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    let result = test_vault_connection(&settings, Some(&db)).await;
+    assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err());
+    println!("✓ Successfully connected to Vault at {}", settings.address);
+}
+
+/// Test migration from database to Vault
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_migrate_db_to_vault(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // First verify we can connect to Vault
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Check initial state - secrets should exist in database
+    let secrets_before = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query secrets");
+
+    println!("Found {} secrets in database before migration:", secrets_before.len());
+    for s in &secrets_before {
+        println!("  - {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+    }
+
+    // Run migration to Vault
+    println!("\nMigrating secrets to Vault...");
+    let report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration to Vault failed");
+
+    println!("Migration report:");
+    println!("  Total secrets: {}", report.total_secrets);
+    println!("  Migrated: {}", report.migrated_count);
+    println!("  Failed: {}", report.failed_count);
+
+    if !report.failures.is_empty() {
+        println!("  Failures:");
+        for f in &report.failures {
+            println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+        }
+    }
+
+    assert_eq!(report.failed_count, 0, "Migration had failures");
+    assert!(report.migrated_count > 0, "No secrets were migrated");
+
+    // Verify secrets are in Vault
+    println!("\nVerifying secrets in Vault...");
+    let vault_backend = VaultBackend::new(settings.clone());
+
+    for secret in &secrets_before {
+        let result: Result<String> = vault_backend
+            .get_secret(&secret.workspace_id, &secret.path)
+            .await;
+        assert!(
+            result.is_ok(),
+            "Failed to read secret {}/{} from Vault: {:?}",
+            secret.workspace_id,
+            secret.path,
+            result.err()
+        );
+        println!("  ✓ {}/{} exists in Vault", secret.workspace_id, secret.path);
+    }
+
+    println!("\n✓ Migration to Vault completed successfully");
+}
+
+/// Test migration from Vault to database
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_migrate_vault_to_db(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // First verify we can connect to Vault
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // First, migrate secrets TO Vault so we have something to migrate back
+    println!("Setting up: migrating secrets to Vault first...");
+    let to_vault_report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Initial migration to Vault failed");
+    assert!(to_vault_report.migrated_count > 0, "No secrets to test with");
+    println!("  Migrated {} secrets to Vault", to_vault_report.migrated_count);
+
+    // Clear the database values to simulate fresh migration back
+    println!("\nClearing database secret values...");
+    sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true")
+        .execute(&db)
+        .await
+        .expect("Failed to clear database values");
+
+    // Verify they were cleared
+    let cleared = sqlx::query!(
+        "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'"
+    )
+    .fetch_one(&db)
+    .await
+    .expect("Failed to count cleared");
+    println!("  Cleared {} secret values in database", cleared.count.unwrap_or(0));
+
+    // Now migrate from Vault back to database
+    println!("\nMigrating secrets from Vault to database...");
+    let report = migrate_secrets_to_database(&db, &settings)
+        .await
+        .expect("Migration to database failed");
+
+    println!("Migration report:");
+    println!("  Total secrets: {}", report.total_secrets);
+    println!("  Migrated: {}", report.migrated_count);
+    println!("  Failed: {}", report.failed_count);
+
+    if !report.failures.is_empty() {
+        println!("  Failures:");
+        for f in &report.failures {
+            println!("    - {}/{}: {}", f.workspace_id, f.path, f.error);
+        }
+    }
+
+    assert_eq!(report.failed_count, 0, "Migration had failures");
+    assert!(report.migrated_count > 0, "No secrets were migrated");
+
+    // Verify secrets are restored in database
+    println!("\nVerifying secrets in database...");
+    let secrets_after = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query restored secrets");
+
+    assert!(
+        !secrets_after.is_empty(),
+        "No secrets were restored in database"
+    );
+
+    for s in &secrets_after {
+        println!("  ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len());
+    }
+
+    println!("\n✓ Migration to database completed successfully");
+}
+
+/// Test full round-trip migration: DB -> Vault -> DB
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_full_round_trip_migration(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    // Verify Vault connection
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Get original secrets
+    let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query original secrets")
+    .into_iter()
+    .map(|r| ((r.workspace_id, r.path), r.value))
+    .collect();
+
+    println!("Original secrets: {} entries", original_secrets.len());
+
+    // Step 1: Migrate to Vault
+    println!("\n=== Step 1: Migrate DB -> Vault ===");
+    let to_vault = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration to Vault failed");
+    println!("Migrated {} secrets to Vault", to_vault.migrated_count);
+    assert_eq!(to_vault.failed_count, 0);
+
+    // Step 2: Clear database values
+    println!("\n=== Step 2: Clear database values ===");
+    sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true")
+        .execute(&db)
+        .await
+        .expect("Failed to clear values");
+
+    // Step 3: Migrate back from Vault
+    println!("\n=== Step 3: Migrate Vault -> DB ===");
+    let to_db = migrate_secrets_to_database(&db, &settings)
+        .await
+        .expect("Migration to database failed");
+    println!("Migrated {} secrets to database", to_db.migrated_count);
+    assert_eq!(to_db.failed_count, 0);
+
+    // Step 4: Verify round-trip integrity
+    println!("\n=== Step 4: Verify round-trip integrity ===");
+    let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!(
+        "SELECT workspace_id, path, value FROM variable WHERE is_secret = true"
+    )
+    .fetch_all(&db)
+    .await
+    .expect("Failed to query restored secrets")
+    .into_iter()
+    .map(|r| ((r.workspace_id, r.path), r.value))
+    .collect();
+
+    // Compare original and restored
+    for ((ws, path), _original_value) in &original_secrets {
+        let restored_value = restored_secrets
+            .get(&(ws.clone(), path.clone()))
+            .expect(&format!("Secret {}/{} not found after round-trip", ws, path));
+
+        // Note: Values might differ slightly due to encryption/decryption
+        // but they should not be the cleared value
+        assert_ne!(
+            restored_value, "ROUND_TRIP_CLEARED",
+            "Secret {}/{} was not restored",
+            ws, path
+        );
+        println!("  ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len());
+    }
+
+    println!("\n✓ Full round-trip migration completed successfully!");
+    println!("  Original secrets: {}", original_secrets.len());
+    println!("  Restored secrets: {}", restored_secrets.len());
+}
+
+/// Test that workspace isolation is maintained during migration
+#[sqlx::test(fixtures("base", "secret_backend"))]
+#[ignore = "requires running Vault instance"]
+async fn test_workspace_isolation(db: Pool<Postgres>) {
+    let settings = test_vault_settings();
+
+    test_vault_connection(&settings, Some(&db))
+        .await
+        .expect("Failed to connect to Vault");
+
+    // Migrate all secrets to Vault
+    let report = migrate_secrets_to_vault(&db, &settings)
+        .await
+        .expect("Migration failed");
+
+    println!("Migrated {} secrets across workspaces", report.migrated_count);
+
+    // Verify workspace isolation in Vault
+    let vault_backend = VaultBackend::new(settings.clone());
+
+    // Try to access test-workspace-2 secret from test-workspace path (should fail)
+    let cross_workspace_result: Result<String> = vault_backend
+        .get_secret("test-workspace", "u/test-user/other_secret")
+        .await;
+
+    assert!(
+        cross_workspace_result.is_err(),
+        "Cross-workspace access should fail - workspace isolation violated!"
+    );
+    println!("✓ Cross-workspace access correctly denied");
+
+    // Verify each workspace's secrets are accessible from their own workspace
+    let ws1_result: Result<String> = vault_backend
+        .get_secret("test-workspace", "u/test-user/db_password")
+        .await;
+    assert!(ws1_result.is_ok(), "test-workspace secret should be accessible");
+    println!("✓ test-workspace secrets accessible");
+
+    let ws2_result: Result<String> = vault_backend
+        .get_secret("test-workspace-2", "u/test-user/other_secret")
+        .await;
+    assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible");
+    println!("✓ test-workspace-2 secrets accessible");
+
+    println!("\n✓ Workspace isolation verified!");
+}

backend/windmill-api/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/lib.rs"
 
 [features]
 default = []
-private = ["windmill-audit/private"]
+private = ["windmill-audit/private", "windmill-common/private"]
 enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
 stripe = []
 agent_worker_server = []

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.610.0
+  version: 1.613.4
   title: Windmill API
 
   contact:
@@ -1411,6 +1411,97 @@ paths:
                 items:
                   $ref: "#/components/schemas/GlobalSetting"
 
+  /min_keep_alive_version:
+    get:
+      summary: get minimum worker version required to stay alive
+      operationId: getMinKeepAliveVersion
+      tags:
+        - setting
+      responses:
+        "200":
+          description: minimum keep-alive version
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /.well-known/jwks.json:
+    get:
+      summary: get JWKS for Vault JWT authentication
+      operationId: getJwks
+      tags:
+        - setting
+      responses:
+        "200":
+          description: JSON Web Key Set
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JwksResponse"
+
+  /settings/test_secret_backend:
+    post:
+      summary: test secret backend connection (HashiCorp Vault)
+      operationId: testSecretBackend
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings to test
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: connection successful
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /settings/migrate_secrets_to_vault:
+    post:
+      summary: migrate secrets from database to HashiCorp Vault
+      operationId: migrateSecretsToVault
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings for migration target
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: migration report
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SecretMigrationReport"
+
+  /settings/migrate_secrets_to_database:
+    post:
+      summary: migrate secrets from HashiCorp Vault to database
+      operationId: migrateSecretsToDatabase
+      tags:
+        - setting
+      requestBody:
+        description: Vault settings for migration source
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/VaultSettings"
+      responses:
+        "200":
+          description: migration report
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SecretMigrationReport"
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -3429,23 +3520,6 @@ paths:
                   default_app_path:
                     type: string
 
-  /w/{workspace}/workspaces/get_large_file_storage_config:
-    get:
-      summary: get large file storage config
-      operationId: getLargeFileStorageConfig
-      tags:
-        - workspace
-      parameters:
-        - $ref: "#/components/parameters/WorkspaceId"
-
-      responses:
-        "200":
-          description: status
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/LargeFileStorage"
-
   /w/{workspace}/workspaces/usage:
     get:
       summary: get usage
@@ -5843,11 +5917,6 @@ paths:
       tags:
         - worker
       parameters:
-        - name: workspace
-          in: query
-          schema:
-            type: string
-          required: false
         - name: show_workspace_restriction
           in: query
           schema:
@@ -5863,6 +5932,24 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/workers/custom_tags:
+    get:
+      summary: get custom tags available for this workspace
+      operationId: getCustomTagsForWorkspace
+      tags:
+        - worker
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: list of custom tags for workspace
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
   /workers/get_default_tags:
     get:
       summary: get all instance default tags
@@ -9254,6 +9341,25 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/jobs/get_otel_traces/{id}:
+    get:
+      summary: get OpenTelemetry traces for a job
+      operationId: getJobOtelTraces
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: list of OTEL Span objects (compatible with OpenTelemetry Span proto)
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+
   /w/{workspace}/trigger/{trigger_kind}/resume_suspended_trigger_jobs/{trigger_path}:
     post:
       summary: resume all suspended jobs for a specific trigger
@@ -16597,6 +16703,83 @@ components:
     # -- INLINE END --
     # Do not change line above
 
+    VaultSettings:
+      type: object
+      required:
+        - address
+        - mount_path
+      properties:
+        address:
+          type: string
+          description: HashiCorp Vault server address (e.g., https://vault.company.com:8200)
+        mount_path:
+          type: string
+          description: KV v2 secrets engine mount path (e.g., windmill)
+        jwt_role:
+          type: string
+          description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used)
+        namespace:
+          type: string
+          description: Vault Enterprise namespace (optional)
+        token:
+          type: string
+          description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication)
+
+    SecretMigrationFailure:
+      type: object
+      required:
+        - workspace_id
+        - path
+        - error
+      properties:
+        workspace_id:
+          type: string
+          description: Workspace ID where the secret is located
+        path:
+          type: string
+          description: Path of the secret that failed to migrate
+        error:
+          type: string
+          description: Error message
+
+    SecretMigrationReport:
+      type: object
+      required:
+        - total_secrets
+        - migrated_count
+        - failed_count
+        - failures
+      properties:
+        total_secrets:
+          type: integer
+          format: int64
+          description: Total number of secrets found
+        migrated_count:
+          type: integer
+          format: int64
+          description: Number of secrets successfully migrated
+        failed_count:
+          type: integer
+          format: int64
+          description: Number of secrets that failed to migrate
+        failures:
+          type: array
+          items:
+            $ref: "#/components/schemas/SecretMigrationFailure"
+          description: Details of any failures encountered during migration
+
+    JwksResponse:
+      type: object
+      required:
+        - keys
+      properties:
+        keys:
+          type: array
+          items:
+            type: object
+            additionalProperties: true
+          description: Array of JSON Web Keys for JWT verification
+
     FlowConversation:
       type: object
       required:
@@ -17397,6 +17580,7 @@ components:
         - visible_to_owner
         - tag
 
+
     ExportableCompletedJob:
       type: object
       description: Completed job with full data for export/import operations
@@ -20786,6 +20970,10 @@ components:
             type: boolean
         custom_path:
           type: string
+        raw_app:
+          type: boolean
+        bundle_secret:
+          type: string
       required:
         - id
         - workspace_id
@@ -20798,6 +20986,7 @@ components:
         - policy
         - execution_mode
         - extra_perms
+        - raw_app
 
     AppWithLastVersionWDraft:
       allOf:

backend/windmill-api/src/ai.rs

@@ -6,7 +6,7 @@ use http::{HeaderMap, Method};
 use quick_cache::sync::Cache;
 use reqwest::{Client, RequestBuilder};
 use serde::{Deserialize, Serialize};
-use serde_json::value::RawValue;
+use serde_json::{json, value::RawValue};
 use std::collections::HashMap;
 use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel};
@@ -125,6 +125,15 @@ struct AIOAuthResource {
     user: Option<String>,
 }
 
+/// Platform for Anthropic API
+#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
+#[serde(rename_all = "snake_case")]
+enum AnthropicPlatform {
+    #[default]
+    Standard,
+    GoogleVertexAi,
+}
+
 #[derive(Deserialize, Debug)]
 struct AIStandardResource {
     #[serde(alias = "baseUrl")]
@@ -137,6 +146,9 @@ struct AIStandardResource {
     aws_access_key_id: Option<String>,
     #[serde(alias = "awsSecretAccessKey")]
     aws_secret_access_key: Option<String>,
+    /// Platform for Anthropic API (standard or google_vertex_ai)
+    #[serde(default)]
+    platform: AnthropicPlatform,
 }
 
 #[derive(Deserialize, Debug)]
@@ -161,6 +173,7 @@ struct AIRequestConfig {
     pub region: Option<String>,
     pub aws_access_key_id: Option<String>,
     pub aws_secret_access_key: Option<String>,
+    pub platform: AnthropicPlatform,
 }
 
 impl AIRequestConfig {
@@ -179,9 +192,11 @@ impl AIRequestConfig {
             region,
             aws_access_key_id,
             aws_secret_access_key,
+            platform,
         ) = match resource {
             AIResource::Standard(resource) => {
                 let region = resource.region.clone();
+                let platform = resource.platform.clone();
                 let base_url = provider
                     .get_base_url(resource.base_url, resource.region, db)
                     .await?;
@@ -216,6 +231,7 @@ impl AIRequestConfig {
                     region,
                     aws_access_key_id,
                     aws_secret_access_key,
+                    platform,
                 )
             }
             AIResource::OAuth(resource) => {
@@ -227,7 +243,17 @@ impl AIRequestConfig {
                 let token = Self::get_token_using_oauth(resource, db, w_id).await?;
                 let base_url = provider.get_base_url(None, None, db).await?;
 
-                (None, Some(token), None, base_url, user, None, None, None)
+                (
+                    None,
+                    Some(token),
+                    None,
+                    base_url,
+                    user,
+                    None,
+                    None,
+                    None,
+                    AnthropicPlatform::Standard,
+                )
             }
         };
 
@@ -240,6 +266,7 @@ impl AIRequestConfig {
             region,
             aws_access_key_id,
             aws_secret_access_key,
+            platform,
         })
     }
 
@@ -294,8 +321,18 @@ impl AIRequestConfig {
 
         let is_azure = provider.is_azure_openai(base_url);
         let is_anthropic = matches!(provider, AIProvider::Anthropic);
+        let is_anthropic_vertex = is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi;
         let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some();
         let is_bedrock = matches!(provider, AIProvider::AWSBedrock);
+        let is_google_ai = matches!(provider, AIProvider::GoogleAI);
+
+        // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent
+        let base_url = if is_google_ai {
+            format!("{}/openai", base_url)
+        } else {
+            base_url.to_string()
+        };
+        let base_url = base_url.as_str();
 
         // Check if using IAM credentials for Bedrock (instead of bearer token)
         let use_iam_auth =
@@ -317,6 +354,10 @@ impl AIRequestConfig {
             let bedrock_base_url = base_url.replace("bedrock-runtime.", "bedrock.");
             let bedrock_url = format!("{}/{}", bedrock_base_url, path);
             (bedrock_url, body)
+        } else if is_anthropic_vertex && method != Method::GET {
+            let (model, transformed_body) = transform_anthropic_for_vertex(&body)?;
+            let vertex_url = format!("{}/{}:streamRawPredict", base_url, model);
+            (vertex_url, transformed_body)
         } else if is_azure {
             let azure_url = AIProvider::build_azure_openai_url(base_url, path);
             (azure_url, body)
@@ -336,7 +377,12 @@ impl AIRequestConfig {
             .header("content-type", "application/json");
 
         for (header_name, header_value) in headers.iter() {
+            // Forward anthropic-* headers, but skip anthropic-version for Vertex AI
+            // (Vertex AI requires anthropic_version in the request body, not as a header)
             if header_name.to_string().starts_with("anthropic-") {
+                if is_anthropic_vertex && header_name.as_str() == "anthropic-version" {
+                    continue;
+                }
                 request = request.header(header_name, header_value);
             }
         }
@@ -360,13 +406,14 @@ impl AIRequestConfig {
             }
         } else {
             // For non-IAM auth, use bearer token or API key
-            if let Some(api_key) = self.api_key {
+            if let Some(api_key) = self.api_key.clone() {
                 if is_azure {
                     request = request.header("api-key", api_key.clone())
                 } else {
                     request = request.header("authorization", format!("Bearer {}", api_key.clone()))
                 }
-                if is_anthropic {
+                // For standard Anthropic API, also add X-API-Key header
+                if is_anthropic && !is_anthropic_vertex {
                     request = request.header("X-API-Key", api_key);
                 }
             }
@@ -438,6 +485,81 @@ pub struct AIConfig {
     pub max_tokens_per_model: Option<HashMap<String, i32>>,
 }
 
+/// Anthropic API version for Google Vertex AI
+const ANTHROPIC_VERSION_VERTEX: &str = "vertex-2023-10-16";
+
+/// Transforms an Anthropic request for Google Vertex AI:
+/// - Extracts the model from the body (needed for the URL)
+/// - Adds anthropic_version to the body
+fn transform_anthropic_for_vertex(body: &Bytes) -> Result<(String, Bytes)> {
+    let mut json_body: HashMap<String, serde_json::Value> = serde_json::from_slice(body)
+        .map_err(|e| Error::internal_err(format!("Failed to parse Anthropic request: {}", e)))?;
+
+    // Extract and remove model from body
+    let model = json_body
+        .remove("model")
+        .and_then(|v| v.as_str().map(|s| s.to_string()))
+        .ok_or_else(|| Error::BadRequest("Missing 'model' field in Anthropic request".to_string()))?;
+
+    // Add anthropic_version to body (required for Vertex AI)
+    json_body.insert(
+        "anthropic_version".to_string(),
+        serde_json::Value::String(ANTHROPIC_VERSION_VERTEX.to_string()),
+    );
+
+    let transformed_body = serde_json::to_vec(&json_body)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize Vertex request: {}", e)))?;
+
+    Ok((model, Bytes::from(transformed_body)))
+}
+
+// FIM (Fill-in-the-Middle) simulation for providers that don't support native FIM
+#[derive(Deserialize, Debug)]
+struct FimRequest {
+    model: String,
+    prompt: String,         // code before cursor
+    suffix: Option<String>, // code after cursor
+    temperature: Option<f32>,
+    max_tokens: Option<u32>,
+    stop: Option<Vec<String>>,
+}
+
+/// Checks if the AI provider supports native FIM (Fill-in-the-Middle) endpoint
+fn supports_native_fim(provider: &AIProvider) -> bool {
+    matches!(provider, AIProvider::Mistral)
+}
+
+/// Transforms a FIM request to chat/completions format for providers that don't support native FIM.
+fn transform_fim_to_chat_completions(body: &Bytes) -> Result<(Bytes, String)> {
+    let fim_req: FimRequest = serde_json::from_slice(body)
+        .map_err(|e| Error::internal_err(format!("Failed to parse FIM request: {}", e)))?;
+
+    let suffix = fim_req.suffix.unwrap_or_default();
+
+    let system_prompt = "You are a code completion assistant. Complete the code at the <CURSOR/> position between the given prefix and suffix. Output ONLY the code that goes at the cursor - no explanations, no markdown, no repeating the prefix or suffix.";
+
+    let user_content = format!(
+        "<PREFIX>\n{}\n<CURSOR/>\n<SUFFIX>\n{}",
+        fim_req.prompt, suffix
+    );
+
+    let chat_req = json!({
+        "model": fim_req.model,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_content}
+        ],
+        "temperature": fim_req.temperature.unwrap_or(0.0),
+        "max_tokens": fim_req.max_tokens.unwrap_or(256),
+        "stop": fim_req.stop
+    });
+
+    let chat_body = serde_json::to_vec(&chat_req)
+        .map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?;
+
+    Ok((Bytes::from(chat_body), "chat/completions".to_string()))
+}
+
 pub fn global_service() -> Router {
     Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy))
 }
@@ -516,10 +638,10 @@ async fn global_proxy(
 async fn proxy(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
-    Path((w_id, ai_path)): Path<(String, String)>,
+    Path((w_id, mut ai_path)): Path<(String, String)>,
     method: Method,
     headers: HeaderMap,
-    body: Bytes,
+    mut body: Bytes,
 ) -> impl IntoResponse {
     let provider = headers
         .get("X-Provider")
@@ -600,6 +722,19 @@ async fn proxy(
         }
     };
 
+    // Check if this is a FIM request to a provider that doesn't support native FIM endpoint
+    // For such providers, transform to use FIM sentinel tokens with the chat/completions endpoint
+    let is_fim_request = ai_path.contains("fim/completions");
+    if is_fim_request && !supports_native_fim(&provider) {
+        tracing::debug!(
+            "Transforming FIM request to chat/completions with FIM tokens for provider {:?}",
+            provider
+        );
+        let (chat_body, chat_path) = transform_fim_to_chat_completions(&body)?;
+        body = chat_body;
+        ai_path = chat_path;
+    }
+
     // Extract model and streaming flag for Bedrock transformation (only for POST requests)
     let (model, is_streaming) =
         if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST {

backend/windmill-api/src/apps.rs

@@ -120,6 +120,7 @@ pub fn unauthed_service() -> Router {
         .route("/download_s3_file/*path", get(download_s3_file_from_app))
         .route("/public_app/:secret", get(get_public_app_by_secret))
         .route("/public_resource/*path", get(get_public_resource))
+        .route("/get_data/v/*id", get(get_raw_app_data))
 }
 pub fn global_service() -> Router {
     Router::new()
@@ -175,6 +176,9 @@ pub struct AppWithLastVersion {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub custom_path: Option<String>,
     pub raw_app: bool,
+    #[sqlx(skip)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bundle_secret: Option<String>,
 }
 
 #[derive(Serialize, FromRow)]
@@ -770,7 +774,7 @@ async fn get_public_app_by_secret(
         "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
         null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
         app_version.created_at, app_version.created_by, app_version.raw_app
-        FROM app, app_version 
+        FROM app, app_version
         LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
         WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
         .bind(&id)
@@ -778,36 +782,37 @@ async fn get_public_app_by_secret(
     .fetch_optional(&db)
     .await?;
 
-    let app = not_found_if_none(app_o, "App", id.to_string())?;
+    let mut app = not_found_if_none(app_o, "App", id.to_string())?;
 
     let policy = serde_json::from_str::<Policy>(app.policy.0.get()).map_err(to_anyhow)?;
 
-    if matches!(policy.execution_mode, ExecutionMode::Anonymous) {
-        return Ok(Json(app));
-    }
-
-    if opt_authed.is_none() {
-        {
+    if !matches!(policy.execution_mode, ExecutionMode::Anonymous) {
+        if opt_authed.is_none() {
             return Err(Error::NotAuthorized(
                 "App visibility does not allow public access and you are not logged in".to_string(),
             ));
+        } else {
+            let authed = opt_authed.unwrap();
+            let mut tx = user_db.begin(&authed).await?;
+            let is_visible = sqlx::query_scalar!(
+                "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)",
+                id,
+                &w_id
+            )
+            .fetch_one(&mut *tx)
+            .await?;
+            tx.commit().await?;
+            if !is_visible.unwrap_or(false) {
+                return Err(Error::NotAuthorized(
+                    "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(),
+                ));
+            }
         }
-    } else {
-        let authed = opt_authed.unwrap();
-        let mut tx = user_db.begin(&authed).await?;
-        let is_visible = sqlx::query_scalar!(
-            "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)",
-            id,
-            &w_id
-        )
-        .fetch_one(&mut *tx)
-        .await?;
-        tx.commit().await?;
-        if !is_visible.unwrap_or(false) {
-            return Err(Error::NotAuthorized(
-                "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(),
-            ));
-        }
+    }
+
+    // Compute bundle_secret for raw apps
+    if app.raw_app {
+        app.bundle_secret = Some(compute_bundle_secret(&db, &w_id, &app.versions).await?);
     }
 
     Ok(Json(app))
@@ -892,6 +897,15 @@ async fn get_secret_id(
 
 const BUNDLE_SECRET_PREFIX: &str = "bundle_";
 
+pub async fn compute_bundle_secret(db: &DB, w_id: &str, versions: &[i64]) -> Result<String> {
+    let version_id = versions
+        .last()
+        .ok_or_else(|| Error::internal_err("App has no versions".to_string()))?;
+    let mc = build_crypt(db, w_id).await?;
+    let hx = hex::encode(mc.encrypt_str_to_bytes(format!("{}{}", BUNDLE_SECRET_PREFIX, version_id)));
+    Ok(hx)
+}
+
 async fn get_latest_version_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -2249,6 +2263,7 @@ async fn upload_s3_file_from_app(
                     );
                     let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths(
                         &db_with_opt_authed,
+                        Some(&on_behalf_authed),
                         &w_id,
                         None,
                         &[(&file_key, S3Permission::WRITE)],
@@ -2280,6 +2295,7 @@ async fn upload_s3_file_from_app(
                 DbWithOptAuthed::from_authed(&on_behalf_authed, db.clone(), None);
             let (_, s3_resource_opt) = get_workspace_s3_resource_and_check_paths(
                 &db_with_opt_authed,
+                Some(&on_behalf_authed),
                 &w_id,
                 None,
                 &[(&file_key, S3Permission::WRITE)],
@@ -2319,6 +2335,7 @@ async fn upload_s3_file_from_app(
                 let db_with_opt_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), None);
                 let (_, s3_resource) = get_workspace_s3_resource_and_check_paths(
                     &db_with_opt_authed,
+                Some(&authed),
                     &w_id,
                     None,
                     &[(&file_key, S3Permission::WRITE)],
@@ -2359,7 +2376,7 @@ async fn upload_s3_file_from_app(
     ])
     .into();
 
-    upload_file_from_req(s3_client, &file_key, request, options).await?;
+    let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?;
 
     let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims {
         file_key: file_key.clone(),
@@ -2429,6 +2446,7 @@ async fn delete_s3_file_from_app(
         let db_with_opt_authed = DbWithOptAuthed::from_authed(&on_behalf_authed, db.clone(), None);
         let (_, s3_resource) = get_workspace_s3_resource_and_check_paths(
             &db_with_opt_authed,
+                Some(&on_behalf_authed),
             &w_id,
             None,
             &[(&path.to_string(), S3Permission::DELETE)],

backend/windmill-api/src/azure_proxy_oss.rs

@@ -0,0 +1,6 @@
+// OSS stub for Azure proxy functionality
+// The actual implementation is in azure_proxy_ee.rs (Enterprise Edition)
+
+#[cfg(all(feature = "private", feature = "parquet"))]
+#[allow(unused)]
+pub use crate::azure_proxy_ee::*;

backend/windmill-api/src/concurrency_groups.rs

@@ -224,6 +224,7 @@ async fn get_concurrent_intervals(
             concurrency_key: Some(_),
             allow_wildcards: None,
             trigger_kind: _,
+            include_args: _,
         } => true,
         _ => false,
     };

backend/windmill-api/src/flows.rs

@@ -34,9 +34,10 @@ use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::runnable_settings::RunnableSettingsTrait;
 use windmill_common::utils::query_elems_from_hub;
-use windmill_common::worker::{
-    to_raw_value, CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
+use windmill_common::min_version::{
+    MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
 };
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::HUB_BASE_URL;
 use windmill_common::{
     db::UserDB,
@@ -1476,14 +1477,14 @@ async fn archive_flow_by_path(
 /// Validates that flow debouncing configuration is supported by all workers
 /// Returns an error if debouncing is configured but workers are behind required version
 async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> {
-    if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await
+    if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await
         && !nf.parse_flow_value()?.debouncing_settings.is_default()
     {
         tracing::warn!(
             "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature"
         );
         Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() })
-    } else if !*MIN_VERSION_SUPPORTS_DEBOUNCING_V2.read().await
+    } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
         && !nf
             .parse_flow_value()?
             .debouncing_settings

backend/windmill-api/src/job_helpers_oss.rs

@@ -12,7 +12,7 @@ use windmill_common::s3_helpers::StorageResourceType;
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use crate::db::{ApiAuthed, DB};
 #[cfg(all(feature = "parquet", not(feature = "private")))]
-use object_store::{ObjectStore, PutMultipartOpts};
+use object_store::{ObjectStore, PutMultipartOpts, PutResult};
 #[cfg(all(feature = "parquet", not(feature = "private")))]
 use std::sync::Arc;
 #[cfg(not(feature = "private"))]
@@ -85,7 +85,7 @@ pub async fn upload_file_from_req(
     _file_key: &str,
     _req: axum::extract::Request,
     _options: PutMultipartOpts,
-) -> error::Result<()> {
+) -> error::Result<PutResult> {
     Err(error::Error::internal_err(
         "Not implemented in Windmill's Open Source repository".to_string(),
     ))
@@ -152,6 +152,7 @@ pub struct DeleteS3FileQuery {
 #[cfg(not(feature = "private"))]
 pub async fn get_workspace_s3_resource_and_check_paths<'c>(
     _db_with_opt_authed: &DbWithOptAuthed<'c, ApiAuthed>,
+    _authed_api: Option<&ApiAuthed>,
     _w_id: &str,
     _storage: Option<String>,
     _paths: &[(&str, windmill_common::s3_helpers::S3Permission)],

backend/windmill-api/src/jobs.rs

@@ -64,7 +64,9 @@ use crate::{
     concurrency_groups::join_concurrency_key,
     db::{ApiAuthed, DB},
     triggers::trigger_helpers::RunnableId,
-    users::{get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed},
+    users::{
+        get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed,
+    },
     utils::{check_scopes, content_plain, require_super_admin},
 };
 use anyhow::Context;
@@ -341,6 +343,7 @@ pub fn workspaced_service() -> Router {
             "/send_email_with_instance_smtp",
             post(send_email_with_instance_smtp),
         )
+        .route("/get_otel_traces/:id", get(get_otel_traces))
 }
 
 pub fn workspace_unauthed_service() -> Router {
@@ -1765,6 +1768,8 @@ pub struct ListableCompletedJob {
     pub priority: Option<i16>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub labels: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub args: Option<serde_json::Value>,
 }
 
 #[derive(Debug, Deserialize, Clone, Default)]
@@ -1852,6 +1857,7 @@ pub struct ListQueueQuery {
     pub allow_wildcards: Option<bool>,
     pub trigger_kind: Option<JobTriggerKind>,
     pub trigger_path: Option<String>,
+    pub include_args: Option<bool>,
 }
 
 impl From<ListCompletedQuery> for ListQueueQuery {
@@ -1885,6 +1891,7 @@ impl From<ListCompletedQuery> for ListQueueQuery {
             allow_wildcards: lcq.allow_wildcards,
             trigger_kind: lcq.trigger_kind,
             trigger_path: lcq.trigger_path,
+            include_args: lcq.include_args,
         }
     }
 }
@@ -2074,6 +2081,16 @@ async fn list_queue_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListQueueQuery>,
 ) -> error::JsonResult<Vec<ListableQueuedJob>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
+    let args_field = if include_args { "v2_job.args" } else { "null as args" };
+
     let sql = list_queue_jobs_query(
         &w_id,
         &lq,
@@ -2086,7 +2103,7 @@ async fn list_queue_jobs(
             "v2_job_queue.scheduled_for",
             "v2_job.runnable_id as script_hash",
             "v2_job.runnable_path as script_path",
-            "null as args",
+            args_field,
             "v2_job.kind as job_kind",
             "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path",
             "v2_job.permissioned_as",
@@ -2413,6 +2430,14 @@ async fn list_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<Job>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
 
@@ -2425,13 +2450,36 @@ async fn list_jobs(
             "cannot specify both success and running".to_string(),
         ));
     }
+
+    // Create dynamic field arrays when include_args is true
+    let cj_fields: Vec<&str>;
+    let qj_fields: Vec<&str>;
+    let cj_fields_ref: &[&str];
+    let qj_fields_ref: &[&str];
+
+    if include_args {
+        cj_fields = UnifiedJob::completed_job_fields()
+            .iter()
+            .map(|f| if *f == "null as args" { "v2_job.args" } else { *f })
+            .collect();
+        qj_fields = UnifiedJob::queued_job_fields()
+            .iter()
+            .map(|f| if *f == "null as args" { "v2_job.args" } else { *f })
+            .collect();
+        cj_fields_ref = &cj_fields;
+        qj_fields_ref = &qj_fields;
+    } else {
+        cj_fields_ref = UnifiedJob::completed_job_fields();
+        qj_fields_ref = UnifiedJob::queued_job_fields();
+    }
+
     let sqlc = if lq.running.is_none() {
         Some(list_completed_jobs_query(
             &w_id,
             Some(per_page),
             0,
             &ListCompletedQuery { order_desc: Some(true), ..lqc },
-            UnifiedJob::completed_job_fields(),
+            cj_fields_ref,
             true,
             get_scope_tags(&authed),
         ))
@@ -2449,7 +2497,7 @@ async fn list_jobs(
         let mut sqlq = list_queue_jobs_query(
             &w_id,
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
-            UnifiedJob::queued_job_fields(),
+            qj_fields_ref,
             Pagination { per_page: None, page: None },
             true,
             get_scope_tags(&authed),
@@ -3164,10 +3212,22 @@ pub async fn get_resume_urls_internal(
             "{base_url}/approve/{w_id}/{target_job_id}/{resume_id}/{signature}{approver_query}"
         ),
         cancel: build_resume_url(
-            "cancel", &w_id, &target_job_id, &resume_id, &signature, &approver_query, &base_url,
+            "cancel",
+            &w_id,
+            &target_job_id,
+            &resume_id,
+            &signature,
+            &approver_query,
+            &base_url,
         ),
         resume: build_resume_url(
-            "resume", &w_id, &target_job_id, &resume_id, &signature, &approver_query, &base_url,
+            "resume",
+            &w_id,
+            &target_job_id,
+            &resume_id,
+            &signature,
+            &approver_query,
+            &base_url,
         ),
     };
 
@@ -3446,6 +3506,7 @@ pub struct UnifiedJob {
     pub running: Option<bool>,
     pub script_hash: Option<ScriptHash>,
     pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
     pub duration_ms: Option<i64>,
     pub success: Option<bool>,
     pub deleted: bool,
@@ -3566,6 +3627,7 @@ impl UnifiedJob {
 
 impl<'a> From<UnifiedJob> for Job {
     fn from(uj: UnifiedJob) -> Self {
+        let args = uj.args.and_then(|v| serde_json::from_value(v).ok());
         match uj.typ.as_ref() {
             "CompletedJob" => Job::CompletedJob(JobExtended::new(
                 uj.self_wait_time_ms,
@@ -3582,7 +3644,7 @@ impl<'a> From<UnifiedJob> for Job {
                     success: uj.success.unwrap(),
                     script_hash: uj.script_hash,
                     script_path: uj.script_path,
-                    args: None,
+                    args: args.clone(),
                     result: None,
                     result_columns: None,
                     logs: None,
@@ -3622,7 +3684,7 @@ impl<'a> From<UnifiedJob> for Job {
                     script_hash: uj.script_hash,
                     script_path: uj.script_path,
                     script_entrypoint_override: None,
-                    args: None,
+                    args,
                     logs: None,
                     canceled: uj.canceled,
                     canceled_by: uj.canceled_by,
@@ -5108,7 +5170,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
                     if success {
                         StatusCode::OK
                     } else {
-                        StatusCode::INTERNAL_SERVER_ERROR
+                        StatusCode::UNPROCESSABLE_ENTITY
                     },
                     Json(result),
                 )
@@ -5122,7 +5184,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
                 })
                 .unwrap_or_else(|| {
                     if !success {
-                        Ok(StatusCode::INTERNAL_SERVER_ERROR)
+                        Ok(StatusCode::UNPROCESSABLE_ENTITY)
                     } else if result_value.is_some() {
                         Ok(StatusCode::OK)
                     } else {
@@ -5172,7 +5234,7 @@ pub fn result_to_response(result: Box<RawValue>, success: bool) -> error::Result
             if success {
                 StatusCode::OK
             } else {
-                StatusCode::INTERNAL_SERVER_ERROR
+                StatusCode::UNPROCESSABLE_ENTITY
             },
             Json(result),
         )
@@ -8331,6 +8393,7 @@ pub struct ListCompletedQuery {
     pub allow_wildcards: Option<bool>,
     pub trigger_kind: Option<JobTriggerKind>,
     pub trigger_path: Option<String>,
+    pub include_args: Option<bool>,
 }
 
 async fn list_completed_jobs(
@@ -8340,8 +8403,18 @@ async fn list_completed_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<ListableCompletedJob>> {
+    let include_args = lq.include_args.unwrap_or(false);
+
+    if include_args && *CLOUD_HOSTED {
+        return Err(error::Error::BadRequest(
+            "include_args is not supported on cloud hosted Windmill".to_string(),
+        ));
+    }
+
     let (per_page, offset) = paginate(pagination);
 
+    let args_field = if include_args { "v2_job.args" } else { "null as args" };
+
     let sql = list_completed_jobs_query(
         &w_id,
         Some(per_page),
@@ -8377,6 +8450,7 @@ async fn list_completed_jobs(
             "v2_job.tag",
             "v2_job.priority",
             "v2_job_completed.result->'wm_labels' as labels",
+            args_field,
             "'CompletedJob' as type",
         ],
         false,
@@ -8776,3 +8850,63 @@ async fn delete_completed_job<'a>(
     )
     .await;
 }
+
+async fn get_otel_traces(
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> error::Result<Json<Vec<serde_json::Value>>> {
+    // Check job exists and user has permission to view it
+    let job = sqlx::query_scalar!(
+        "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
+        id,
+        w_id
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    match job {
+        Some(created_by) => {
+            if opt_authed.is_none() && created_by != "anonymous" {
+                return Err(Error::BadRequest(
+                    "As a non logged in user, you can only see jobs ran by anonymous users"
+                        .to_string(),
+                ));
+            }
+        }
+        None => {
+            return Err(Error::NotFound(format!("Job {} not found", id)));
+        }
+    }
+
+    let trace_id = id.as_bytes().as_slice();
+
+    let traces = sqlx::query_scalar!(
+        r#"SELECT json_build_object(
+            'trace_id', encode(trace_id, 'hex'),           -- BYTEA to hex string
+            'span_id', encode(span_id, 'hex'),             -- BYTEA to hex string
+            'parent_span_id', encode(parent_span_id, 'hex'), -- BYTEA to hex string
+            'trace_state', trace_state,
+            'flags', flags,
+            'name', name,
+            'kind', kind,
+            'start_time_unix_nano', start_time_unix_nano,
+            'end_time_unix_nano', end_time_unix_nano,
+            'attributes', attributes,
+            'dropped_attributes_count', dropped_attributes_count,
+            'events', events,
+            'dropped_events_count', dropped_events_count,
+            'links', links,
+            'dropped_links_count', dropped_links_count,
+            'status', status
+        ) as "span!"
+        FROM otel_traces
+        WHERE trace_id = $1
+        ORDER BY start_time_unix_nano"#,
+        trace_id
+    )
+    .fetch_all(&db)
+    .await?;
+
+    Ok(Json(traces))
+}

backend/windmill-api/src/lib.rs

@@ -78,6 +78,9 @@ pub mod args;
 mod assets;
 mod audit;
 pub mod auth;
+#[cfg(all(feature = "private", feature = "parquet"))]
+pub mod azure_proxy_ee;
+mod azure_proxy_oss;
 mod bedrock;
 mod capture;
 mod concurrency_groups;
@@ -148,6 +151,7 @@ pub mod scim_ee;
 mod scim_oss;
 mod scopes;
 mod scripts;
+mod secret_backend_ext;
 mod service_logs;
 mod settings;
 mod slack_approvals;
@@ -410,13 +414,15 @@ pub async fn run_server(
     let (mcp_router, mcp_cancellation_token) = {
         #[cfg(feature = "mcp")]
         if server_mode || mcp_mode {
+            use mcp::add_www_authenticate_header;
             let (mcp_router, mcp_cancellation_token) =
                 setup_mcp_server(db.clone(), user_db).await?;
-            let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id);
-            (
-                mcp_router.layer(mcp_middleware),
-                Some(mcp_cancellation_token),
-            )
+            // Apply middleware: auth check inside WWW-Authenticate wrapper so 401s get the header
+            let mcp_router = mcp_router
+                .route_layer(from_extractor::<ApiAuthed>())
+                .layer(axum::middleware::from_fn(add_www_authenticate_header))
+                .layer(axum::middleware::from_fn(extract_and_store_workspace_id));
+            (mcp_router, Some(mcp_cancellation_token))
         } else {
             (Router::new(), None)
         }
@@ -496,6 +502,16 @@ pub async fn run_server(
                             #[cfg(not(feature = "oauth2"))]
                             Router::new()
                         })
+                        .nest("/mcp/oauth/server", {
+                            #[cfg(feature = "mcp")]
+                            {
+                                // Only /approve requires authentication (called by frontend)
+                                mcp::oauth_server::workspaced_authed_service()
+                            }
+
+                            #[cfg(not(feature = "mcp"))]
+                            Router::new()
+                        })
                         .nest("/ai", ai::workspaced_service())
                         .nest("/npm_proxy", npm_proxy::workspaced_service())
                         .nest("/raw_apps", raw_apps::workspaced_service())
@@ -507,6 +523,7 @@ pub async fn run_server(
                             users::workspaced_service().layer(Extension(argon2.clone())),
                         )
                         .nest("/variables", variables::workspaced_service())
+                        .nest("/workers", workers::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
                         .nest("/oidc", oidc_oss::workspaced_service())
                         .nest("/openapi", {
@@ -540,10 +557,20 @@ pub async fn run_server(
                 .nest("/embeddings", embeddings::global_service())
                 .nest("/ai", ai::global_service())
                 .nest("/inkeep", inkeep_oss::global_service())
-                .nest("/mcp/w/:workspace_id/sse", mcp_router)
                 .nest("/mcp/w/:workspace_id/list_tools", mcp_list_tools_service)
                 .route_layer(from_extractor::<ApiAuthed>())
                 .route_layer(from_extractor::<users::Tokened>())
+                // Workspace-scoped OAuth endpoints that don't require authentication
+                // (authorize and token are called by MCP client before user is authenticated)
+                .nest("/w/:workspace_id/mcp/oauth/server", {
+                    #[cfg(feature = "mcp")]
+                    {
+                        mcp::oauth_server::workspaced_unauthed_service()
+                    }
+
+                    #[cfg(not(feature = "mcp"))]
+                    Router::new()
+                })
                 .nest("/jobs", jobs::global_root_service())
                 .nest(
                     "/srch/w/:workspace_id/index",
@@ -582,6 +609,9 @@ pub async fn run_server(
                         .layer(cors.clone()),
                 )
                 .layer(from_extractor::<OptAuthed>())
+                // Deprecated, here for backwards compatibility: user should use /mcp/w/:workspace_id/mcp instead
+                .nest("/mcp/w/:workspace_id/sse", mcp_router.clone())
+                .nest("/mcp/w/:workspace_id/mcp", mcp_router)
                 .nest("/agent_workers", {
                     #[cfg(feature = "agent_worker_server")]
                     {
@@ -684,6 +714,15 @@ pub async fn run_server(
                     #[cfg(not(feature = "mcp"))]
                     Router::new()
                 })
+                .nest("/mcp/oauth/server", {
+                    #[cfg(feature = "mcp")]
+                    {
+                        mcp::oauth_server::global_service().layer(cors.clone())
+                    }
+
+                    #[cfg(not(feature = "mcp"))]
+                    Router::new()
+                })
                 .nest("/r", {
                     #[cfg(feature = "http_trigger")]
                     {
@@ -714,11 +753,44 @@ pub async fn run_server(
                     }
                 })
                 .route("/version", get(git_v))
+                .route("/min_keep_alive_version", get(min_keep_alive_version))
                 .route("/uptodate", get(is_up_to_date))
                 .route("/ee_license", get(ee_license))
                 .route("/openapi.yaml", get(openapi))
                 .route("/openapi.json", get(openapi_json)),
         )
+        // Clients must use workspace-scoped OAuth metadata at:
+        // /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server
+        // This is discovered via /.well-known/oauth-protected-resource?workspace_id=...
+        .route(
+            "/.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server",
+            {
+                #[cfg(feature = "mcp")]
+                {
+                    get(mcp::oauth_server::workspaced_oauth_metadata)
+                }
+                #[cfg(not(feature = "mcp"))]
+                {
+                    get(|| async { axum::http::StatusCode::NOT_FOUND })
+                }
+            },
+        )
+        // RFC 9728 path-based discovery: /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp
+        .route(
+            "/.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp",
+            {
+                #[cfg(feature = "mcp")]
+                {
+                    get(mcp::oauth_server::protected_resource_metadata_by_path)
+                }
+                #[cfg(not(feature = "mcp"))]
+                {
+                    get(|| async { axum::http::StatusCode::NOT_FOUND })
+                }
+            },
+        )
+        // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix)
+        .route("/.well-known/jwks.json", get(settings::get_jwks))
         .fallback(static_assets::static_handler)
         .layer(middleware_stack);
 
@@ -833,6 +905,11 @@ async fn git_v() -> String {
     format!("CE {GIT_VERSION}")
 }
 
+async fn min_keep_alive_version() -> String {
+    let v = windmill_common::min_version::MIN_KEEP_ALIVE_VERSION;
+    format!("{}.{}.{}", v.0, v.1, v.2)
+}
+
 #[cfg(not(feature = "enterprise"))]
 async fn ee_license() -> &'static str {
     ""

backend/windmill-api/src/live_migrations.rs

@@ -10,7 +10,7 @@ use sqlx::Postgres;
 use std::time::Duration;
 use tokio::task::JoinHandle;
 use windmill_common::error::Error;
-use windmill_common::worker::MIN_VERSION_IS_AT_LEAST_1_461;
+use windmill_common::min_version::MIN_VERSION_IS_AT_LEAST_1_461;
 
 use crate::db::{CustomMigrator, DB};
 use sqlx::migrate::Migrate;
@@ -36,7 +36,7 @@ pub async fn custom_migrations(
         let db2 = db.clone();
         let v2jh = tokio::task::spawn(async move {
             loop {
-                if !*MIN_VERSION_IS_AT_LEAST_1_461.read().await {
+                if !MIN_VERSION_IS_AT_LEAST_1_461.met().await {
                     tracing::info!("Waiting for all workers to be at least version 1.461 before applying v2 finalize migration, sleeping for 5s...");
                     tokio::time::sleep(Duration::from_secs(5)).await;
                     continue;

backend/windmill-api/src/mcp/core.rs

@@ -411,6 +411,53 @@ pub async fn extract_and_store_workspace_id(
     next.run(request).await
 }
 
+/// Middleware that adds WWW-Authenticate header to 401 responses
+/// This helps MCP clients discover the OAuth authorization server (RFC 9728)
+pub async fn add_www_authenticate_header(
+    request: Request<axum::body::Body>,
+    next: Next,
+) -> Response {
+    use axum::http::StatusCode;
+    use windmill_common::BASE_URL;
+
+    // Extract workspace_id before consuming the request
+    let Some(workspace_id) = request
+        .extensions()
+        .get::<WorkspaceId>()
+        .map(|w| w.0.clone())
+    else {
+        return Response::builder()
+            .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
+            .body(axum::body::Body::from("Missing workspace_id in request"))
+            .unwrap();
+    };
+
+    let response = next.run(request).await;
+
+    // Only add header to 401 Unauthorized responses
+    if response.status() == StatusCode::UNAUTHORIZED {
+        let base_url = BASE_URL.read().await;
+
+        // RFC 9728: The resource parameter contains the protected resource URL.
+        // Clients derive the metadata URL by inserting /.well-known/oauth-protected-resource
+        // after the host, e.g., http://host/.well-known/oauth-protected-resource/api/mcp/w/test/mcp
+        let resource_url = format!("{}/api/mcp/w/{}/mcp", base_url, workspace_id);
+        let www_authenticate = format!("Bearer resource=\"{}\"", resource_url);
+
+        // Reconstruct response with the new header
+        let (mut parts, body) = response.into_parts();
+        parts.headers.insert(
+            axum::http::header::WWW_AUTHENTICATE,
+            www_authenticate
+                .parse()
+                .unwrap_or_else(|_| "Bearer".parse().unwrap()),
+        );
+        Response::from_parts(parts, body)
+    } else {
+        response
+    }
+}
+
 /// Setup the MCP server with HTTP transport
 pub async fn setup_mcp_server(
     db: DB,

backend/windmill-api/src/mcp/mod.rs

@@ -8,4 +8,8 @@ mod core;
 mod utils;
 
 // Re-export only what's needed externally
-pub use core::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server};
+pub mod oauth_server;
+pub use core::{
+    add_www_authenticate_header, extract_and_store_workspace_id, list_tools_service,
+    setup_mcp_server,
+};

backend/windmill-api/src/mcp/oauth_server.rs

@@ -0,0 +1,840 @@
+//! OAuth 2.0 Authorization Server for MCP (RFC 6749, 7591, 7636, 8414, 9728)
+
+use axum::{
+    extract::{Extension, Path, Query},
+    response::{IntoResponse, Redirect},
+    routing::{get, post},
+    Form, Json, Router,
+};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use sqlx::FromRow;
+use windmill_common::{
+    error::{Error, Result},
+    utils::rd_string,
+    BASE_URL, DB,
+};
+
+use crate::db::ApiAuthed;
+
+/// Token expiration for MCP OAuth tokens (1 week in seconds)
+const MCP_OAUTH_TOKEN_EXPIRATION_SECS: u64 = 7 * 24 * 60 * 60;
+
+/// Refresh token expiration for MCP OAuth (30 days in seconds)
+const MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS: u64 = 30 * 24 * 60 * 60;
+
+/// RFC 8414
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AuthorizationMetadata {
+    pub issuer: String,
+    pub authorization_endpoint: String,
+    pub token_endpoint: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub registration_endpoint: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scopes_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub response_types_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub grant_types_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub code_challenge_methods_supported: Option<Vec<String>>,
+}
+
+/// RFC 9728
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ProtectedResourceMetadata {
+    pub resource: String,
+    pub authorization_servers: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scopes_supported: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bearer_methods_supported: Option<Vec<String>>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct OAuthJsonError {
+    pub error: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_description: Option<String>,
+}
+
+impl OAuthJsonError {
+    fn new(error: &str, description: Option<&str>) -> Self {
+        Self { error: error.to_string(), error_description: description.map(|s| s.to_string()) }
+    }
+}
+
+impl IntoResponse for OAuthJsonError {
+    fn into_response(self) -> axum::response::Response {
+        (axum::http::StatusCode::BAD_REQUEST, Json(self)).into_response()
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct OAuthTokenError {
+    pub error: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error_description: Option<String>,
+}
+
+impl OAuthTokenError {
+    fn invalid_request(description: &str) -> Self {
+        Self {
+            error: "invalid_request".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn invalid_grant(description: &str) -> Self {
+        Self {
+            error: "invalid_grant".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn unsupported_grant_type(description: &str) -> Self {
+        Self {
+            error: "unsupported_grant_type".to_string(),
+            error_description: Some(description.to_string()),
+        }
+    }
+
+    fn server_error(description: &str) -> Self {
+        Self { error: "server_error".to_string(), error_description: Some(description.to_string()) }
+    }
+}
+
+impl IntoResponse for OAuthTokenError {
+    fn into_response(self) -> axum::response::Response {
+        let status = match self.error.as_str() {
+            "server_error" => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+            _ => axum::http::StatusCode::BAD_REQUEST,
+        };
+        (status, Json(self)).into_response()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ClientRegistrationRequest {
+    pub client_name: String,
+    pub redirect_uris: Vec<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ClientRegistrationResponse {
+    pub client_id: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub client_secret: Option<String>,
+    pub client_name: String,
+    pub redirect_uris: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AuthorizeQuery {
+    pub response_type: String,
+    pub client_id: String,
+    pub redirect_uri: String,
+    #[serde(default)]
+    pub scope: Option<String>,
+    #[serde(default)]
+    pub state: Option<String>,
+    #[serde(default)]
+    pub code_challenge: Option<String>,
+    #[serde(default)]
+    pub code_challenge_method: Option<String>,
+    #[serde(default)]
+    pub resource: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ApprovalForm {
+    pub client_id: String,
+    pub redirect_uri: String,
+    pub scope: String,
+    pub state: String,
+    pub code_challenge: String,
+    pub code_challenge_method: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TokenRequest {
+    pub grant_type: String,
+    #[serde(default)]
+    pub code: String,
+    #[serde(default)]
+    pub redirect_uri: String,
+    #[serde(default)]
+    pub client_id: String,
+    #[serde(default)]
+    pub code_verifier: Option<String>,
+    #[serde(default)]
+    pub refresh_token: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TokenResponse {
+    pub access_token: String,
+    pub token_type: String,
+    pub expires_in: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scope: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub refresh_token: Option<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct OAuthClient {
+    client_id: String,
+    client_name: String,
+    redirect_uris: Vec<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct AuthorizationCode {
+    code: String,
+    client_id: String,
+    user_email: String,
+    workspace_id: String,
+    scopes: Vec<String>,
+    redirect_uri: String,
+    code_challenge: Option<String>,
+    code_challenge_method: Option<String>,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, FromRow)]
+struct RefreshTokenRow {
+    id: i64,
+    refresh_token: String,
+    access_token: String,
+    client_id: String,
+    user_email: String,
+    workspace_id: String,
+    scopes: Vec<String>,
+    token_family: sqlx::types::Uuid,
+    created_at: chrono::DateTime<chrono::Utc>,
+    expires_at: chrono::DateTime<chrono::Utc>,
+    used_at: Option<chrono::DateTime<chrono::Utc>>,
+    revoked: bool,
+}
+
+fn supported_scopes() -> Vec<String> {
+    vec![
+        "mcp:all".to_string(),
+        "mcp:favorites".to_string(),
+        "mcp:scripts:*".to_string(),
+        "mcp:flows:*".to_string(),
+        "mcp:endpoints:*".to_string(),
+    ]
+}
+
+/// GET /.well-known/oauth-authorization-server/api/w/:workspace_id/mcp/oauth/server
+pub async fn workspaced_oauth_metadata(
+    Path(workspace_id): Path<String>,
+) -> Json<AuthorizationMetadata> {
+    let base_url = BASE_URL.read().await;
+    let issuer = format!("{}/api/w/{}/mcp/oauth/server", base_url, workspace_id);
+
+    Json(AuthorizationMetadata {
+        issuer,
+        authorization_endpoint: format!(
+            "{}/api/w/{}/mcp/oauth/server/authorize",
+            base_url, workspace_id
+        ),
+        token_endpoint: format!("{}/api/w/{}/mcp/oauth/server/token", base_url, workspace_id),
+        registration_endpoint: Some(format!("{}/api/mcp/oauth/server/register", base_url)),
+        scopes_supported: Some(supported_scopes()),
+        response_types_supported: Some(vec!["code".to_string()]),
+        grant_types_supported: Some(vec![
+            "authorization_code".to_string(),
+            "refresh_token".to_string(),
+        ]),
+        code_challenge_methods_supported: Some(vec!["S256".to_string()]),
+    })
+}
+
+/// GET /.well-known/oauth-protected-resource/api/mcp/w/:workspace_id/mcp
+pub async fn protected_resource_metadata_by_path(
+    Path(workspace_id): Path<String>,
+) -> Json<ProtectedResourceMetadata> {
+    let base_url = BASE_URL.read().await;
+    let resource_url = format!("{}/api/mcp/w/{}/mcp", base_url, workspace_id);
+    let auth_server_url = format!("{}/api/w/{}/mcp/oauth/server", base_url, workspace_id);
+    Json(ProtectedResourceMetadata {
+        resource: resource_url,
+        authorization_servers: vec![auth_server_url],
+        scopes_supported: Some(supported_scopes()),
+        bearer_methods_supported: Some(vec!["header".to_string()]),
+    })
+}
+
+/// POST /api/mcp/oauth/server/register - dynamic client registration
+pub async fn oauth_register(
+    Extension(db): Extension<DB>,
+    Json(req): Json<ClientRegistrationRequest>,
+) -> Result<(axum::http::StatusCode, Json<ClientRegistrationResponse>)> {
+    if req.redirect_uris.is_empty() {
+        return Err(Error::BadRequest(
+            "At least one redirect_uri is required".to_string(),
+        ));
+    }
+
+    let client_id = format!("mcp-client-{}", rd_string(16));
+
+    sqlx::query!(
+        "INSERT INTO mcp_oauth_server_client (client_id, client_name, redirect_uris)
+         VALUES ($1, $2, $3)",
+        client_id,
+        req.client_name,
+        &req.redirect_uris,
+    )
+    .execute(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Failed to register client: {}", e)))?;
+
+    Ok((
+        axum::http::StatusCode::CREATED,
+        Json(ClientRegistrationResponse {
+            client_id,
+            client_secret: None,
+            client_name: req.client_name,
+            redirect_uris: req.redirect_uris,
+        }),
+    ))
+}
+
+#[derive(Debug, Serialize)]
+pub struct ApprovalResponse {
+    pub code: String,
+    pub state: Option<String>,
+}
+
+/// POST /api/w/:workspace_id/mcp/oauth/server/token - exchange code for token or refresh
+pub async fn oauth_token(
+    Extension(db): Extension<DB>,
+    Form(req): Form<TokenRequest>,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    match req.grant_type.as_str() {
+        "authorization_code" => handle_authorization_code_grant(&db, &req).await,
+        "refresh_token" => handle_refresh_token_grant(&db, &req).await,
+        _ => Err(OAuthTokenError::unsupported_grant_type(
+            "Supported grant types: authorization_code, refresh_token",
+        )),
+    }
+}
+
+/// Handle authorization_code grant type
+async fn handle_authorization_code_grant(
+    db: &DB,
+    req: &TokenRequest,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    let auth_code = match sqlx::query_as!(
+        AuthorizationCode,
+        "DELETE FROM mcp_oauth_server_code
+         WHERE code = $1 AND expires_at > now()
+         RETURNING code, client_id, user_email, workspace_id, scopes, redirect_uri,
+                   code_challenge, code_challenge_method",
+        req.code
+    )
+    .fetch_optional(db)
+    .await
+    {
+        Ok(Some(code)) => code,
+        Ok(None) => {
+            return Err(OAuthTokenError::invalid_grant(
+                "Invalid or expired authorization code",
+            ));
+        }
+        Err(e) => {
+            tracing::error!("Database error consuming auth code: {}", e);
+            return Err(OAuthTokenError::server_error("Database error"));
+        }
+    };
+
+    if auth_code.client_id != req.client_id {
+        return Err(OAuthTokenError::invalid_grant("client_id mismatch"));
+    }
+
+    if auth_code.redirect_uri != req.redirect_uri {
+        return Err(OAuthTokenError::invalid_grant("redirect_uri mismatch"));
+    }
+
+    let challenge = auth_code.code_challenge.as_ref().ok_or_else(|| {
+        OAuthTokenError::invalid_grant("Authorization code missing PKCE challenge")
+    })?;
+
+    let verifier = req
+        .code_verifier
+        .as_ref()
+        .ok_or_else(|| OAuthTokenError::invalid_request("code_verifier is required"))?;
+
+    let method = auth_code.code_challenge_method.as_deref().unwrap_or("S256");
+    if method != "S256" {
+        return Err(OAuthTokenError::invalid_grant(
+            "Only S256 PKCE method is supported",
+        ));
+    }
+
+    if !validate_pkce_s256(verifier, challenge) {
+        return Err(OAuthTokenError::invalid_grant("Invalid code_verifier"));
+    }
+
+    let access_token = rd_string(32);
+    let refresh_token = rd_string(32);
+    let token_family = sqlx::types::Uuid::new_v4();
+    let scopes = auth_code.scopes;
+
+    // Create access token
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)
+         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+        access_token,
+        auth_code.user_email,
+        format!("mcp-oauth-{}", auth_code.client_id),
+        MCP_OAUTH_TOKEN_EXPIRATION_SECS.to_string(),
+        &scopes,
+        auth_code.workspace_id,
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create access token: {}", e);
+        return Err(OAuthTokenError::server_error(
+            "Failed to create access token",
+        ));
+    }
+
+    // Create refresh token
+    let refresh_token_result = sqlx::query!(
+        "INSERT INTO mcp_oauth_refresh_token
+         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+        refresh_token,
+        access_token,
+        auth_code.client_id,
+        auth_code.user_email,
+        auth_code.workspace_id,
+        &scopes,
+        token_family,
+        MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
+    )
+    .execute(db)
+    .await;
+
+    let refresh_token_value = match refresh_token_result {
+        Ok(_) => Some(refresh_token),
+        Err(e) => {
+            tracing::error!("Failed to create refresh token: {}", e);
+            None // Don't include invalid refresh token
+        }
+    };
+
+    Ok(Json(TokenResponse {
+        access_token,
+        token_type: "Bearer".to_string(),
+        expires_in: MCP_OAUTH_TOKEN_EXPIRATION_SECS,
+        scope: Some(scopes.join(" ")),
+        refresh_token: refresh_token_value,
+    }))
+}
+
+/// Handle refresh_token grant type with token rotation and theft detection
+async fn handle_refresh_token_grant(
+    db: &DB,
+    req: &TokenRequest,
+) -> std::result::Result<Json<TokenResponse>, OAuthTokenError> {
+    let refresh_token_value = req
+        .refresh_token
+        .as_ref()
+        .ok_or_else(|| OAuthTokenError::invalid_request("refresh_token is required"))?;
+
+    if req.client_id.is_empty() {
+        return Err(OAuthTokenError::invalid_request("client_id is required"));
+    }
+
+    // Atomically claim the refresh token by setting used_at in a single UPDATE.
+    let token_row = match sqlx::query_as!(
+        RefreshTokenRow,
+        "UPDATE mcp_oauth_refresh_token
+         SET used_at = now()
+         WHERE refresh_token = $1
+           AND client_id = $2
+           AND used_at IS NULL
+           AND NOT revoked
+           AND expires_at > now()
+         RETURNING id, refresh_token, access_token, client_id, user_email, workspace_id,
+                   scopes, token_family, created_at, expires_at, used_at, revoked",
+        refresh_token_value,
+        req.client_id
+    )
+    .fetch_optional(db)
+    .await
+    {
+        Ok(Some(row)) => row,
+        Ok(None) => {
+            // Check for token reuse (theft detection) and revoke family if detected
+            if let Ok(Some(family)) = sqlx::query_scalar!(
+                "SELECT token_family FROM mcp_oauth_refresh_token
+                 WHERE refresh_token = $1 AND used_at IS NOT NULL",
+                refresh_token_value
+            )
+            .fetch_optional(db)
+            .await
+            {
+                tracing::warn!("Refresh token reuse detected, revoking family {:?}", family);
+                let _ = sqlx::query!(
+                    "UPDATE mcp_oauth_refresh_token SET revoked = TRUE WHERE token_family = $1",
+                    family
+                )
+                .execute(db)
+                .await;
+            }
+            return Err(OAuthTokenError::invalid_grant("Invalid refresh token"));
+        }
+        Err(e) => {
+            tracing::error!("Database error claiming refresh token: {}", e);
+            return Err(OAuthTokenError::server_error("Database error"));
+        }
+    };
+
+    // Delete old access token
+    if let Err(e) = sqlx::query!("DELETE FROM token WHERE token = $1", token_row.access_token)
+        .execute(db)
+        .await
+    {
+        tracing::error!("Failed to delete old access token: {}", e);
+        // Non-fatal, continue with token creation
+    }
+
+    // Generate new tokens
+    let new_access_token = rd_string(32);
+    let new_refresh_token = rd_string(32);
+    let scopes = token_row.scopes;
+
+    // Create new access token
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO token (token, email, label, expiration, scopes, workspace_id)
+         VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5, $6)",
+        new_access_token,
+        token_row.user_email,
+        format!("mcp-oauth-{}", token_row.client_id),
+        MCP_OAUTH_TOKEN_EXPIRATION_SECS.to_string(),
+        &scopes,
+        token_row.workspace_id,
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create new access token: {}", e);
+        return Err(OAuthTokenError::server_error(
+            "Failed to create access token",
+        ));
+    }
+
+    // Create new refresh token (same token family for tracking)
+    if let Err(e) = sqlx::query!(
+        "INSERT INTO mcp_oauth_refresh_token
+         (refresh_token, access_token, client_id, user_email, workspace_id, scopes, token_family, expires_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, now() + ($8 || ' seconds')::interval)",
+        new_refresh_token,
+        new_access_token,
+        token_row.client_id,
+        token_row.user_email,
+        token_row.workspace_id,
+        &scopes,
+        token_row.token_family,
+        MCP_OAUTH_REFRESH_TOKEN_EXPIRATION_SECS.to_string(),
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Failed to create new refresh token: {}", e);
+        // Access token was created, return success without refresh token
+    }
+
+    Ok(Json(TokenResponse {
+        access_token: new_access_token,
+        token_type: "Bearer".to_string(),
+        expires_in: MCP_OAUTH_TOKEN_EXPIRATION_SECS,
+        scope: Some(scopes.join(" ")),
+        refresh_token: Some(new_refresh_token),
+    }))
+}
+
+/// GET /api/w/:workspace_id/mcp/oauth/server/authorize - redirects to consent page
+pub async fn workspaced_oauth_authorize(
+    Extension(db): Extension<DB>,
+    Path(workspace_id): Path<String>,
+    Query(params): Query<AuthorizeQuery>,
+) -> impl IntoResponse {
+    let client = match sqlx::query_as!(
+        OAuthClient,
+        "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+        params.client_id
+    )
+    .fetch_optional(&db)
+    .await
+    {
+        Ok(Some(client)) => client,
+        Ok(None) => {
+            return OAuthJsonError::new("invalid_client", Some("Unknown client_id"))
+                .into_response();
+        }
+        Err(e) => {
+            tracing::error!("Database error looking up client: {}", e);
+            return OAuthJsonError::new("server_error", Some("Database error"))
+                .into_response();
+        }
+    };
+
+    if !client.redirect_uris.contains(&params.redirect_uri) {
+        return OAuthJsonError::new(
+            "invalid_request",
+            Some("redirect_uri does not match registered URIs"),
+        )
+        .into_response();
+    }
+
+    if params.response_type != "code" {
+        return OAuthErrorRedirect::new(
+            &params.redirect_uri,
+            "unsupported_response_type",
+            Some("Only 'code' response type is supported"),
+            params.state.as_deref(),
+        )
+        .into_response();
+    }
+
+    let code_challenge = match &params.code_challenge {
+        Some(challenge) if !challenge.is_empty() => challenge.as_str(),
+        _ => {
+            return OAuthErrorRedirect::new(
+                &params.redirect_uri,
+                "invalid_request",
+                Some("PKCE required: code_challenge parameter is mandatory"),
+                params.state.as_deref(),
+            )
+            .into_response();
+        }
+    };
+
+    let code_challenge_method = params.code_challenge_method.as_deref().unwrap_or("S256");
+
+    if code_challenge_method != "S256" {
+        return OAuthErrorRedirect::new(
+            &params.redirect_uri,
+            "invalid_request",
+            Some("Invalid code_challenge_method: only 'S256' is supported"),
+            params.state.as_deref(),
+        )
+        .into_response();
+    }
+
+    let resource = match &params.resource {
+        Some(r) => r,
+        None => {
+            return OAuthErrorRedirect::new(
+                &params.redirect_uri,
+                "invalid_request",
+                Some("Missing 'resource' parameter. Required for MCP audience binding."),
+                params.state.as_deref(),
+            )
+            .into_response();
+        }
+    };
+
+    let base_url = BASE_URL.read().await;
+    let frontend_url = format!(
+        "{}/oauth/mcp_authorize?{}",
+        base_url,
+        serde_urlencoded::to_string(&[
+            ("workspace_id", workspace_id.as_str()),
+            ("client_id", params.client_id.as_str()),
+            ("client_name", client.client_name.as_str()),
+            ("redirect_uri", params.redirect_uri.as_str()),
+            ("scope", params.scope.as_deref().unwrap_or("mcp:all")),
+            ("state", params.state.as_deref().unwrap_or("")),
+            ("code_challenge", code_challenge),
+            ("code_challenge_method", code_challenge_method),
+            ("resource", resource),
+        ])
+        .unwrap_or_default()
+    );
+
+    Redirect::temporary(&frontend_url).into_response()
+}
+
+/// POST /api/w/:workspace_id/mcp/oauth/server/approve - user approval (frontend)
+pub async fn workspaced_oauth_approve(
+    Extension(db): Extension<DB>,
+    Path(workspace_id): Path<String>,
+    authed: ApiAuthed,
+    Json(form): Json<ApprovalForm>,
+) -> Result<Json<ApprovalResponse>> {
+    // Verify user is a member of the workspace
+    let is_member = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)",
+        workspace_id,
+        authed.email
+    )
+    .fetch_one(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
+    .unwrap_or(false);
+
+    if !is_member {
+        return Err(Error::NotAuthorized(
+            "User is not a member of this workspace".to_string(),
+        ));
+    }
+
+    // Verify client exists and redirect_uri is registered
+    let client = sqlx::query_as!(
+        OAuthClient,
+        "SELECT client_id, client_name, redirect_uris FROM mcp_oauth_server_client WHERE client_id = $1",
+        form.client_id
+    )
+    .fetch_optional(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Database error: {}", e)))?
+    .ok_or_else(|| Error::BadRequest("Unknown client_id".to_string()))?;
+
+    if !client.redirect_uris.contains(&form.redirect_uri) {
+        return Err(Error::BadRequest(
+            "Invalid redirect_uri for this client".to_string(),
+        ));
+    }
+
+    if form.code_challenge.is_empty() {
+        return Err(Error::BadRequest(
+            "PKCE required: code_challenge is mandatory".to_string(),
+        ));
+    }
+
+    if form.code_challenge_method.is_empty() {
+        return Err(Error::BadRequest(
+            "PKCE required: code_challenge_method is mandatory".to_string(),
+        ));
+    }
+
+    if form.code_challenge_method != "S256" {
+        return Err(Error::BadRequest(
+            "Invalid code_challenge_method: only 'S256' is supported".to_string(),
+        ));
+    }
+
+    let code = format!("mcp-code-{}", rd_string(32));
+
+    let scopes: Vec<String> = form
+        .scope
+        .split_whitespace()
+        .map(|s| s.to_string())
+        .collect();
+
+    sqlx::query!(
+        "INSERT INTO mcp_oauth_server_code
+         (code, client_id, user_email, workspace_id, scopes, redirect_uri, code_challenge, code_challenge_method)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
+        code,
+        form.client_id,
+        authed.email,
+        workspace_id,
+        &scopes,
+        form.redirect_uri,
+        &form.code_challenge,
+        &form.code_challenge_method,
+    )
+    .execute(&db)
+    .await
+    .map_err(|e| Error::InternalErr(format!("Failed to store authorization code: {}", e)))?;
+
+    Ok(Json(ApprovalResponse {
+        code,
+        state: if form.state.is_empty() {
+            None
+        } else {
+            Some(form.state)
+        },
+    }))
+}
+
+/// PKCE validation (S256 only)
+fn validate_pkce_s256(verifier: &str, challenge: &str) -> bool {
+    let mut hasher = Sha256::new();
+    hasher.update(verifier.as_bytes());
+    let computed = base64_url_encode(&hasher.finalize());
+    constant_time_eq(computed.as_bytes(), challenge.as_bytes())
+}
+
+/// Base64 URL encoding (no padding)
+fn base64_url_encode(data: &[u8]) -> String {
+    use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
+    URL_SAFE_NO_PAD.encode(data)
+}
+
+/// Constant-time comparison
+fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    a.iter().zip(b.iter()).fold(0, |acc, (x, y)| acc | (x ^ y)) == 0
+}
+
+/// Helper for OAuth error redirects
+struct OAuthErrorRedirect {
+    redirect_uri: String,
+    error: String,
+    error_description: Option<String>,
+    state: Option<String>,
+}
+
+impl OAuthErrorRedirect {
+    fn new(
+        redirect_uri: &str,
+        error: &str,
+        error_description: Option<&str>,
+        state: Option<&str>,
+    ) -> Self {
+        Self {
+            redirect_uri: redirect_uri.to_string(),
+            error: error.to_string(),
+            error_description: error_description.map(|s| s.to_string()),
+            state: state.map(|s| s.to_string()),
+        }
+    }
+}
+
+impl IntoResponse for OAuthErrorRedirect {
+    fn into_response(self) -> axum::response::Response {
+        let mut url = format!("{}?error={}", self.redirect_uri, self.error);
+        if let Some(desc) = &self.error_description {
+            url.push_str(&format!("&error_description={}", urlencoding::encode(desc)));
+        }
+        if let Some(state) = &self.state {
+            url.push_str(&format!("&state={}", state));
+        }
+        Redirect::temporary(&url).into_response()
+    }
+}
+
+/// Mounted at /api/mcp/oauth/server
+pub fn global_service() -> Router {
+    Router::new().route("/register", post(oauth_register))
+}
+
+/// Workspace-scoped OAuth endpoints that don't require authentication
+/// Mounted at /api/w/:workspace_id/mcp/oauth/server (outside authenticated section)
+pub fn workspaced_unauthed_service() -> Router {
+    Router::new()
+        .route("/authorize", get(workspaced_oauth_authorize))
+        .route("/token", post(oauth_token))
+}
+
+/// Workspace-scoped OAuth endpoints that require authentication
+/// Mounted at /api/w/:workspace_id/mcp/oauth/server (inside authenticated section)
+pub fn workspaced_authed_service() -> Router {
+    Router::new().route("/approve", post(workspaced_oauth_approve))
+}

backend/windmill-api/src/resources.rs

@@ -10,6 +10,7 @@ use std::collections::HashMap;
 
 use crate::{
     db::{ApiAuthed, DB},
+    secret_backend_ext::rename_vault_secret,
     users::{maybe_refresh_folders, require_owner_of_path, Tokened},
     utils::{check_scopes, require_super_admin, BulkDeleteRequest},
     var_resource_cache::{cache_resource, get_cached_resource},
@@ -947,12 +948,40 @@ async fn update_resource(
 
     let mut tx = user_db.begin(&authed).await?;
 
-    if let Some(npath) = ns.path {
+    if let Some(npath) = ns.path.clone() {
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
 
             require_owner_of_path(&authed, path)?;
 
+            // Handle Vault secret rename if the linked variable is a Vault-stored secret
+            let linked_var = sqlx::query!(
+                "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+
+            if let Some(var) = linked_var {
+                if var.is_secret {
+                    // Check if this is a Vault-stored secret and rename it
+                    if let Some(new_value) =
+                        rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
+                    {
+                        // Update the variable's value to point to the new Vault path
+                        sqlx::query!(
+                            "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+                            new_value,
+                            path,
+                            w_id
+                        )
+                        .execute(&mut *tx)
+                        .await?;
+                    }
+                }
+            }
+
             sqlx::query!(
                 "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
                 npath,

backend/windmill-api/src/scripts.rs

@@ -49,7 +49,8 @@ use windmill_common::{
     s3_helpers::upload_artifact_to_store,
     scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline},
     utils::{paginate_without_limits, WarnAfterExt},
-    worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2},
+    min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2},
+    worker::CLOUD_HOSTED,
 };
 
 use windmill_common::{
@@ -2300,12 +2301,12 @@ async fn delete_scripts_bulk(
 /// Validates that script debouncing configuration is supported by all workers
 /// Returns an error if debouncing is configured but workers are behind required version
 async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> {
-    if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await && !ns.debouncing_settings.is_default() {
+    if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await && !ns.debouncing_settings.is_default() {
         tracing::warn!(
             "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature"
         );
         Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() })
-    } else if !*MIN_VERSION_SUPPORTS_DEBOUNCING_V2.read().await
+    } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
         && !ns.debouncing_settings.is_legacy_compatible()
         && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT
     {

backend/windmill-api/src/secret_backend_ext.rs

@@ -0,0 +1,408 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+//! Secret backend extension for the API layer
+//!
+//! This module provides helper functions for integrating the SecretBackend
+//! trait with variable operations in the API.
+//!
+//! Note: HashiCorp Vault integration requires Enterprise Edition.
+//! The OSS version only supports the database backend.
+
+use std::sync::Arc;
+
+use windmill_common::{
+    db::DB,
+    error::{Error, Result},
+    secret_backend::{database::DatabaseBackend, SecretBackend},
+    variables::{build_crypt, decrypt, encrypt},
+};
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use windmill_common::{
+    global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING},
+    secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings},
+};
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use tokio::sync::RwLock;
+
+// Cached Vault backend to avoid recreating it for every request
+// This enables connection pooling and avoids repeated setup overhead
+#[cfg(all(feature = "private", feature = "enterprise"))]
+struct CachedVaultBackend {
+    backend: Arc<dyn SecretBackend>,
+    settings: VaultSettings,
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+lazy_static::lazy_static! {
+    static ref VAULT_BACKEND_CACHE: RwLock<Option<CachedVaultBackend>> = RwLock::new(None);
+}
+
+/// Get the current secret backend based on global settings
+///
+/// OSS: Always returns DatabaseBackend
+/// EE: Returns configured backend (Database or Vault)
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
+    Ok(Arc::new(DatabaseBackend::new(db.clone())))
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn get_secret_backend(db: &DB) -> Result<Arc<dyn SecretBackend>> {
+    let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
+        Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
+        None => SecretBackendConfig::default(),
+    };
+
+    match config {
+        SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))),
+        SecretBackendConfig::HashiCorpVault(settings) => {
+            get_or_create_vault_backend(db, settings).await
+        }
+    }
+}
+
+/// Get a cached Vault backend or create a new one if settings changed
+#[cfg(all(feature = "private", feature = "enterprise"))]
+async fn get_or_create_vault_backend(
+    _db: &DB,
+    settings: VaultSettings,
+) -> Result<Arc<dyn SecretBackend>> {
+    // Check if we have a cached backend with matching settings (read lock)
+    {
+        let cache = VAULT_BACKEND_CACHE.read().await;
+        if let Some(ref cached) = *cache {
+            if cached.settings == settings {
+                return Ok(cached.backend.clone());
+            }
+        }
+    }
+
+    // Need to create a new backend - acquire write lock
+    let mut cache = VAULT_BACKEND_CACHE.write().await;
+
+    // Double-check (another task may have created it while we waited)
+    if let Some(ref cached) = *cache {
+        if cached.settings == settings {
+            return Ok(cached.backend.clone());
+        }
+    }
+
+    // Create new backend
+    let backend: Arc<dyn SecretBackend> = {
+        #[cfg(feature = "openidconnect")]
+        if settings.token.is_none() {
+            Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone()))
+        } else {
+            Arc::new(VaultBackend::new(settings.clone()))
+        }
+
+        #[cfg(not(feature = "openidconnect"))]
+        Arc::new(VaultBackend::new(settings.clone()))
+    };
+
+    // Cache it
+    *cache = Some(CachedVaultBackend {
+        backend: backend.clone(),
+        settings,
+    });
+
+    Ok(backend)
+}
+
+/// Check if a Vault backend is currently configured
+///
+/// OSS: Always returns false
+/// EE: Checks global settings
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn is_vault_backend_configured(_db: &DB) -> Result<bool> {
+    Ok(false)
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn is_vault_backend_configured(db: &DB) -> Result<bool> {
+    let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? {
+        Some(value) => serde_json::from_value::<SecretBackendConfig>(value).unwrap_or_default(),
+        None => SecretBackendConfig::default(),
+    };
+
+    Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_)))
+}
+
+/// Get a secret value using the configured backend
+///
+/// For database backend: decrypts using workspace key
+/// For vault backend (EE only): fetches from Vault directly
+pub async fn get_secret_value(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+    encrypted_value: &str,
+) -> Result<String> {
+    let backend = get_secret_backend(db).await?;
+
+    match backend.backend_name() {
+        "database" => {
+            // Use existing database decryption
+            let mc = build_crypt(db, workspace_id).await?;
+            decrypt(&mc, encrypted_value.to_string()).map_err(|e| {
+                Error::internal_err(format!("Error decrypting variable {}: {}", path, e))
+            })
+        }
+        "hashicorp_vault" => {
+            // Fetch from Vault directly
+            backend.get_secret(workspace_id, path).await
+        }
+        _ => Err(Error::internal_err(format!(
+            "Unknown backend: {}",
+            backend.backend_name()
+        ))),
+    }
+}
+
+/// Store a secret value using the configured backend
+///
+/// For database backend: encrypts using workspace key and returns encrypted value
+/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage
+pub async fn store_secret_value(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+    plain_value: &str,
+) -> Result<String> {
+    let backend = get_secret_backend(db).await?;
+
+    match backend.backend_name() {
+        "database" => {
+            // Use existing database encryption
+            let mc = build_crypt(db, workspace_id).await?;
+            Ok(encrypt(&mc, plain_value))
+        }
+        "hashicorp_vault" => {
+            // Store in Vault and return a marker for DB
+            backend.set_secret(workspace_id, path, plain_value).await?;
+            // Return a marker indicating the value is stored in Vault
+            // The actual value in the DB will be this marker
+            Ok(format!("$vault:{}", path))
+        }
+        _ => Err(Error::internal_err(format!(
+            "Unknown backend: {}",
+            backend.backend_name()
+        ))),
+    }
+}
+
+/// Delete a secret from the configured backend (if using Vault)
+///
+/// For database backend: no-op (DB delete is handled separately)
+/// For vault backend (EE only): deletes from Vault
+pub async fn delete_secret_from_backend(
+    db: &DB,
+    workspace_id: &str,
+    path: &str,
+) -> Result<()> {
+    if is_vault_backend_configured(db).await? {
+        let backend = get_secret_backend(db).await?;
+        // Ignore NotFound errors during deletion (secret might not exist in Vault)
+        match backend.delete_secret(workspace_id, path).await {
+            Ok(()) => Ok(()),
+            Err(Error::NotFound(_)) => Ok(()),
+            Err(e) => Err(e),
+        }
+    } else {
+        Ok(())
+    }
+}
+
+/// Check if a value is stored in Vault (indicated by the $vault: prefix)
+pub fn is_vault_stored_value(value: &str) -> bool {
+    value.starts_with("$vault:")
+}
+
+/// Rename a secret in Vault when a variable path changes (EE only)
+///
+/// This function:
+/// 1. Reads the secret value from the old path
+/// 2. Writes it to the new path
+/// 3. Deletes from the old path
+/// 4. Returns the new marker value ($vault:new_path)
+///
+/// If the value is not a Vault-stored value, returns None (no action needed).
+/// If Vault is not configured, returns None.
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn rename_vault_secret(
+    _db: &DB,
+    _workspace_id: &str,
+    _old_path: &str,
+    new_path: &str,
+    current_value: &str,
+) -> Result<Option<String>> {
+    // OSS: If value has $vault: prefix, just update the reference
+    // (This handles edge case where EE was used before downgrading to OSS)
+    if is_vault_stored_value(current_value) {
+        tracing::warn!(
+            "Variable has $vault: prefix but Vault requires Enterprise Edition. \
+             Updating DB reference to {}",
+            new_path
+        );
+        return Ok(Some(format!("$vault:{}", new_path)));
+    }
+    Ok(None)
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn rename_vault_secret(
+    db: &DB,
+    workspace_id: &str,
+    old_path: &str,
+    new_path: &str,
+    current_value: &str,
+) -> Result<Option<String>> {
+    // Only handle Vault-stored values
+    if !is_vault_stored_value(current_value) {
+        return Ok(None);
+    }
+
+    // Check if Vault backend is configured
+    if !is_vault_backend_configured(db).await? {
+        // Vault not configured but value has $vault: prefix - this is an inconsistent state
+        // Log warning and return new marker to at least update the DB reference
+        tracing::warn!(
+            "Variable value has $vault: prefix but Vault is not configured. \
+             Updating DB reference from {} to {}",
+            old_path,
+            new_path
+        );
+        return Ok(Some(format!("$vault:{}", new_path)));
+    }
+
+    let backend = get_secret_backend(db).await?;
+
+    // Read from old path
+    let secret_value = match backend.get_secret(workspace_id, old_path).await {
+        Ok(value) => value,
+        Err(Error::NotFound(_)) => {
+            // Secret doesn't exist in Vault - just update the DB reference
+            tracing::warn!(
+                "Secret not found in Vault at path {} during rename to {}",
+                old_path,
+                new_path
+            );
+            return Ok(Some(format!("$vault:{}", new_path)));
+        }
+        Err(e) => return Err(e),
+    };
+
+    // Write to new path
+    backend
+        .set_secret(workspace_id, new_path, &secret_value)
+        .await?;
+
+    // Delete from old path (ignore errors - new path is already written)
+    if let Err(e) = backend.delete_secret(workspace_id, old_path).await {
+        tracing::warn!(
+            "Failed to delete old secret at {} after rename to {}: {}",
+            old_path,
+            new_path,
+            e
+        );
+    }
+
+    Ok(Some(format!("$vault:{}", new_path)))
+}
+
+/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename)
+/// EE only feature.
+///
+/// This is used when renaming users where many secrets need their paths updated.
+/// Returns a list of (old_path, new_value) pairs for updating the database.
+#[cfg(not(all(feature = "private", feature = "enterprise")))]
+pub async fn rename_vault_secrets_with_prefix(
+    _db: &DB,
+    _workspace_id: &str,
+    _old_prefix: &str,
+    _new_prefix: &str,
+    _variables: Vec<(String, String)>,
+) -> Result<Vec<(String, String)>> {
+    // OSS: No Vault support, return empty
+    Ok(vec![])
+}
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn rename_vault_secrets_with_prefix(
+    db: &DB,
+    workspace_id: &str,
+    old_prefix: &str,
+    new_prefix: &str,
+    variables: Vec<(String, String)>, // (path, value) pairs
+) -> Result<Vec<(String, String)>> {
+    // Only process if Vault is configured
+    if !is_vault_backend_configured(db).await? {
+        return Ok(vec![]);
+    }
+
+    let backend = get_secret_backend(db).await?;
+    let mut updates = Vec::new();
+
+    for (old_path, value) in variables {
+        // Only handle Vault-stored values
+        if !is_vault_stored_value(&value) {
+            continue;
+        }
+
+        // Calculate new path by replacing prefix
+        let new_path = if old_path.starts_with(old_prefix) {
+            format!("{}{}", new_prefix, &old_path[old_prefix.len()..])
+        } else {
+            continue; // Path doesn't match prefix, skip
+        };
+
+        // Read from old path
+        let secret_value = match backend.get_secret(workspace_id, &old_path).await {
+            Ok(v) => v,
+            Err(Error::NotFound(_)) => {
+                // Just update DB reference
+                updates.push((old_path, format!("$vault:{}", new_path)));
+                continue;
+            }
+            Err(e) => {
+                tracing::error!(
+                    "Failed to read secret at {} during bulk rename: {}",
+                    old_path,
+                    e
+                );
+                continue;
+            }
+        };
+
+        // Write to new path
+        if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await {
+            tracing::error!(
+                "Failed to write secret to {} during bulk rename: {}",
+                new_path,
+                e
+            );
+            continue;
+        }
+
+        // Delete from old path
+        if let Err(e) = backend.delete_secret(workspace_id, &old_path).await {
+            tracing::warn!(
+                "Failed to delete old secret at {} after rename: {}",
+                old_path,
+                e
+            );
+        }
+
+        updates.push((old_path, format!("$vault:{}", new_path)));
+    }
+
+    Ok(updates)
+}

backend/windmill-api/src/settings.rs

@@ -31,6 +31,8 @@ use crate::utils::require_devops_role;
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "enterprise")]
 use windmill_common::ee_oss::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
+#[cfg(all(feature = "private", feature = "enterprise"))]
+use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings};
 use windmill_common::{
     email_oss::send_email_plain_text,
     error::{self, JsonResult, Result},
@@ -85,6 +87,16 @@ pub fn global_service() -> Router {
             post(acknowledge_all_critical_alerts),
         );
 
+    // Vault integration routes (EE only - requires both private and enterprise features)
+    #[cfg(all(feature = "private", feature = "enterprise"))]
+    let r = r
+        .route("/test_secret_backend", post(test_secret_backend))
+        .route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault))
+        .route(
+            "/migrate_secrets_to_database",
+            post(migrate_secrets_to_database),
+        );
+
     #[cfg(feature = "parquet")]
     {
         return r.route("/test_object_storage_config", post(test_s3_bucket));
@@ -736,7 +748,11 @@ async fn setup_custom_instance_pg_database_inner(
             "Cannot use reserved PostgreSQL database names".to_string(),
         ));
     }
-    if wmill_pg_creds.dbname.trim().eq_ignore_ascii_case(dbname.trim()) {
+    if wmill_pg_creds
+        .dbname
+        .trim()
+        .eq_ignore_ascii_case(dbname.trim())
+    {
         return Err(error::Error::BadRequest(
             "Database name cannot be the same as the main database".to_string(),
         ));
@@ -751,10 +767,7 @@ async fn setup_custom_instance_pg_database_inner(
     .await?
     .unwrap_or(false);
 
-    let pg_creds = PgDatabase {
-        dbname: dbname.to_string(),
-        ..wmill_pg_creds
-    };
+    let pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds };
 
     logs.created_database = "SKIP".to_string();
     if !db_exists {
@@ -777,7 +790,8 @@ async fn setup_custom_instance_pg_database_inner(
              GRANT CREATE ON SCHEMA public TO custom_instance_user;
              GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user;
              ALTER DEFAULT PRIVILEGES IN SCHEMA public
-                 GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;"
+                 GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;
+             ALTER ROLE custom_instance_user CREATEROLE;"
         ))
         .await
         .map_err(|e| {
@@ -798,3 +812,100 @@ async fn setup_custom_instance_pg_database_inner(
 
     Ok(())
 }
+
+// ============================================================================
+// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition
+// ============================================================================
+
+/// Test connection to a secret backend (HashiCorp Vault)
+///
+/// This endpoint validates that the Vault settings are correct and that
+/// Windmill can successfully authenticate and communicate with Vault.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn test_secret_backend(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?;
+
+    Ok("Successfully connected to HashiCorp Vault".to_string())
+}
+
+/// Migrate existing secrets from database to HashiCorp Vault
+///
+/// This endpoint reads all encrypted secrets from the database, decrypts them,
+/// and stores them in HashiCorp Vault. The database values are NOT deleted
+/// automatically to allow for rollback if needed.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn migrate_secrets_to_vault(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> JsonResult<SecretMigrationReport> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?;
+
+    Ok(Json(report))
+}
+
+/// Migrate secrets from HashiCorp Vault back to database
+///
+/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using
+/// the workspace encryption keys, and stores them in the database. The Vault
+/// values are NOT deleted automatically to allow for rollback if needed.
+///
+/// This is an Enterprise Edition feature.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub async fn migrate_secrets_to_database(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(settings): Json<VaultSettings>,
+) -> JsonResult<SecretMigrationReport> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let report =
+        windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?;
+
+    Ok(Json(report))
+}
+
+// ============================================================================
+// JWKS Endpoint for Vault JWT Authentication
+// ============================================================================
+
+/// JSON Web Key Set response structure
+#[derive(Serialize)]
+pub struct JwksResponse {
+    pub keys: Vec<serde_json::Value>,
+}
+
+/// JWKS endpoint for HashiCorp Vault to validate JWTs
+///
+/// Vault calls this endpoint to fetch the public keys used to verify
+/// JWTs generated by Windmill for authentication.
+///
+/// In the open-source version, this returns an empty JWKS.
+/// The Enterprise Edition provides the actual key set.
+pub async fn get_jwks() -> JsonResult<JwksResponse> {
+    // Open source version returns empty JWKS
+    // Enterprise Edition will override this with actual public keys
+    #[cfg(not(feature = "enterprise"))]
+    {
+        Ok(Json(JwksResponse { keys: vec![] }))
+    }
+
+    #[cfg(feature = "enterprise")]
+    {
+        // In enterprise mode, the actual keys would be fetched from global settings
+        // For now, return empty - the EE implementation would override this
+        Ok(Json(JwksResponse { keys: vec![] }))
+    }
+}

backend/windmill-api/src/users.rs

@@ -19,6 +19,7 @@ use crate::db::ApiAuthed;
 
 pub use crate::auth::Tokened;
 
+use crate::secret_backend_ext::rename_vault_secrets_with_prefix;
 use crate::utils::{
     generate_instance_wide_unique_username, get_instance_username_or_create_pending,
 };
@@ -2661,6 +2662,7 @@ async fn rename_user(
         }
         update_username_in_workpsace(
             &mut tx,
+            &db,
             &user_email,
             &w_u.username,
             &ru.new_username,
@@ -2688,6 +2690,7 @@ async fn rename_user(
 
 async fn update_username_in_workpsace<'c>(
     tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    db: &DB,
     email: &str,
     old_username: &str,
     new_username: &str,
@@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>(
 
     // ---- variables ----
 
+    // Handle Vault secret renames before updating paths in DB
+    let old_prefix = format!("u/{}/", old_username);
+    let new_prefix = format!("u/{}/", new_username);
+
+    // Fetch all Vault-stored secret variables under this user's path
+    let vault_secrets: Vec<(String, String)> = sqlx::query!(
+        r#"SELECT path, value FROM variable
+           WHERE path LIKE ('u/' || $1 || '/%')
+           AND workspace_id = $2
+           AND is_secret = true
+           AND value LIKE '$vault:%'"#,
+        old_username,
+        w_id
+    )
+    .fetch_all(&mut **tx)
+    .await?
+    .into_iter()
+    .map(|r| (r.path, r.value))
+    .collect();
+
+    // Rename secrets in Vault and get the new values
+    let vault_updates =
+        rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?;
+
+    // Update the values in the DB for renamed Vault secrets (using OLD path, before path update)
+    for (old_path, new_value) in vault_updates {
+        sqlx::query!(
+            "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3",
+            new_value,
+            old_path,
+            w_id
+        )
+        .execute(&mut **tx)
+        .await?;
+    }
+
+    // Now update the paths in the database
     sqlx::query!(
         r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
         new_username,

backend/windmill-api/src/variables.rs

@@ -8,6 +8,10 @@
 
 use crate::{
     db::{ApiAuthed, DB},
+    secret_backend_ext::{
+        delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret,
+        store_secret_value,
+    },
     users::{maybe_refresh_folders, require_owner_of_path},
     utils::{check_scopes, BulkDeleteRequest},
     webhook_util::{WebhookMessage, WebhookShared},
@@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable};
 use lazy_static::lazy_static;
 use serde::Deserialize;
 use sqlx::{Acquire, Postgres, Transaction};
-use windmill_common::variables::{decrypt, encrypt};
+use windmill_common::variables::encrypt;
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 
 lazy_static! {
@@ -210,13 +214,8 @@ async fn get_variable(
                 return Err(Error::internal_err("Require oauth2 feature".to_string()));
             } else if !value.is_empty() && decrypt_secret {
                 let _ = tx.commit().await;
-                let mc = build_crypt(&db, &w_id).await?;
-                Some(decrypt(&mc, value).map_err(|e| {
-                    Error::internal_err(format!(
-                        "Error decrypting variable {}: {}",
-                        variable.path, e
-                    ))
-                })?)
+                // Use secret backend for decryption (supports both DB and Vault)
+                Some(get_secret_value(&db, &w_id, &variable.path, &value).await?)
             } else if q.include_encrypted.unwrap_or(false) {
                 Some(value)
             } else {
@@ -355,8 +354,8 @@ async fn create_variable(
 
     check_path_conflict(&db, &w_id, &variable.path).await?;
     let value = if variable.is_secret && !already_encrypted.unwrap_or(false) {
-        let mc = build_crypt(&db, &w_id).await?;
-        encrypt(&mc, &variable.value)
+        // Use secret backend for encryption (supports both DB and Vault)
+        store_secret_value(&db, &w_id, &variable.path, &variable.value).await?
     } else {
         variable.value
     };
@@ -436,6 +435,16 @@ async fn delete_variable(
 
     check_scopes(&authed, || format!("variables:write:{}", path))?;
 
+    // Check if variable is a secret before deleting (for Vault cleanup)
+    let is_secret = sqlx::query_scalar!(
+        "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+        path,
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .unwrap_or(false);
+
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
@@ -465,6 +474,11 @@ async fn delete_variable(
 
     tx.commit().await?;
 
+    // If variable was a secret, also delete from Vault backend (if configured)
+    if is_secret {
+        delete_secret_from_backend(&db, &w_id, path).await?;
+    }
+
     handle_deployment_metadata(
         &authed.email,
         &authed.username,
@@ -496,6 +510,15 @@ async fn delete_variables_bulk(
         check_scopes(&authed, || format!("variables:write:{}", path))?;
     }
 
+    // Query which paths are secrets before deletion (for Vault cleanup)
+    let secret_paths: Vec<String> = sqlx::query_scalar!(
+        "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true",
+        &request.paths,
+        &w_id
+    )
+    .fetch_all(&db)
+    .await?;
+
     let mut tx = user_db.begin(&authed).await?;
 
     let deleted_paths = sqlx::query_scalar!(
@@ -526,6 +549,13 @@ async fn delete_variables_bulk(
 
     tx.commit().await?;
 
+    // Delete secrets from Vault backend (if configured)
+    for path in &secret_paths {
+        if deleted_paths.contains(path) {
+            delete_secret_from_backend(&db, &w_id, path).await?;
+        }
+    }
+
     try_join_all(deleted_paths.iter().map(|path| {
         handle_deployment_metadata(
             &authed.email,
@@ -589,7 +619,9 @@ async fn update_variable(
         sqlb.set_str("path", npath);
     }
     let ns_value_is_none = ns.value.is_none();
-    if let Some(nvalue) = ns.value {
+    // Determine the target path for storing secrets (use new path if provided)
+    let target_path = ns.path.as_deref().unwrap_or(path);
+    if let Some(nvalue) = ns.value.clone() {
         let is_secret = if ns.is_secret.is_some() {
             ns.is_secret.unwrap()
         } else {
@@ -604,8 +636,9 @@ async fn update_variable(
         };
 
         let value = if is_secret && !already_encrypted.unwrap_or(false) {
-            let mc = build_crypt(&db, &w_id).await?;
-            encrypt(&mc, &nvalue)
+            // Use secret backend for encryption (supports both DB and Vault)
+            // Store at target_path (new path if renaming, otherwise current path)
+            store_secret_value(&db, &w_id, target_path, &nvalue).await?
         } else {
             nvalue
         };
@@ -654,11 +687,38 @@ async fn update_variable(
 
     let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?;
 
-    if let Some(npath) = ns.path {
+    if let Some(npath) = ns.path.clone() {
         if npath != path {
             check_path_conflict(&db, &w_id, &npath).await?;
             require_owner_of_path(&authed, path)?;
 
+            // Handle Vault secret rename if the variable is a secret stored in Vault
+            let current_var = sqlx::query!(
+                "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2",
+                path,
+                w_id
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+
+            if let Some(var) = current_var {
+                if var.is_secret && is_vault_stored_value(&var.value) {
+                    if ns.value.is_some() {
+                        // New value was provided and already stored at new path
+                        // Just delete the old secret from Vault
+                        delete_secret_from_backend(&db, &w_id, path).await?;
+                    } else {
+                        // No new value - rename the secret in Vault
+                        if let Some(new_value) =
+                            rename_vault_secret(&db, &w_id, path, &npath, &var.value).await?
+                        {
+                            // Update the variable's value to point to the new Vault path
+                            sqlb.set_str("value", &new_value);
+                        }
+                    }
+                }
+            }
+
             let mut v = sqlx::query_scalar!(
                 "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
                 path,
@@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>(
             #[cfg(not(feature = "oauth2"))]
             return Err(Error::internal_err("Require oauth2 feature".to_string()));
         } else if !value.is_empty() {
-            let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?;
-            decrypt(&mc, value).map_err(|e| {
-                Error::internal_err(format!(
-                    "Error decrypting variable {}: {}",
-                    variable.path, e
-                ))
-            })?
+            // Use secret backend for decryption (supports both DB and Vault)
+            get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await?
         } else {
             "".to_string()
         }

backend/windmill-api/src/workers.rs

@@ -7,7 +7,7 @@
  */
 
 use axum::{
-    extract::{Extension, Query},
+    extract::{Extension, Path, Query},
     routing::get,
     Json, Router,
 };
@@ -18,7 +18,7 @@ use uuid::Uuid;
 use windmill_common::{
     db::UserDB,
     error::JsonResult,
-    jobs::TAGS_ARE_SENSITIVE,
+    jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE},
     utils::{paginate, Pagination},
     worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE},
     DB,
@@ -41,6 +41,10 @@ pub fn global_service() -> Router {
         .route("/queue_running_counts", get(get_queue_running_counts))
 }
 
+pub fn workspaced_service() -> Router {
+    Router::new().route("/custom_tags", get(get_custom_tags_for_workspace))
+}
+
 #[derive(FromRow, Serialize, Deserialize)]
 struct WorkerPing {
     worker: String,
@@ -93,6 +97,9 @@ async fn list_worker_pings(
     Query(query): Query<ListWorkerQuery>,
 ) -> JsonResult<Vec<WorkerPing>> {
     let is_super_admin = require_super_admin(&db, &authed.email).await.is_ok();
+    if *HIDE_WORKERS_FOR_NON_ADMINS && !is_super_admin {
+        return Ok(Json(vec![]));
+    }
     let mut tx = user_db.begin(&authed).await?;
 
     let (per_page, offset) = paginate(Pagination { page: query.page, per_page: query.per_page });
@@ -189,24 +196,15 @@ async fn exists_workers_with_tags(
 
 #[derive(Deserialize)]
 struct CustomTagQuery {
-    workspace: Option<String>,
     show_workspace_restriction: Option<bool>,
 }
+
 async fn get_custom_tags(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Query(query): Query<CustomTagQuery>,
 ) -> JsonResult<Vec<String>> {
-    if query.show_workspace_restriction.is_some_and(|x| x) && query.workspace.is_some() {
-        return Err(windmill_common::error::Error::BadRequest(
-            "Cannot use both workspace and show_workspace_restriction".to_string(),
-        ));
-    }
-    if let Some(workspace) = query.workspace {
-        let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
-        let all_tags = tags_o.to_string_vec(Some(workspace));
-        return Ok(Json(all_tags));
-    } else if query.show_workspace_restriction.is_some_and(|x| x) {
+    if query.show_workspace_restriction.is_some_and(|x| x) {
         let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
         let all_tags = tags_o.to_string_vec(None);
         return Ok(Json(all_tags));
@@ -220,6 +218,15 @@ async fn get_custom_tags(
     Ok(Json(ALL_TAGS.read().await.clone().into()))
 }
 
+async fn get_custom_tags_for_workspace(
+    _authed: ApiAuthed,
+    Path(w_id): Path<String>,
+) -> JsonResult<Vec<String>> {
+    let tags_o = CUSTOM_TAGS_PER_WORKSPACE.read().await;
+    let all_tags = tags_o.to_string_vec(Some(w_id));
+    Ok(Json(all_tags))
+}
+
 async fn get_default_tags_per_workspace() -> JsonResult<bool> {
     Ok(Json(
         DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed),

backend/windmill-common/Cargo.toml

@@ -15,7 +15,7 @@ benchmark = []
 parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"]
 aws_auth = ["dep:aws-sdk-sts"]
 otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
-    "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
+    "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic", "dep:opentelemetry"]
 smtp = ["dep:mail-send"]
 scoped_cache = []
 cloud = []
@@ -81,6 +81,7 @@ mail-send = { workspace = true, optional = true }
 futures-core.workspace = true
 async-stream.workspace = true
 const_format.workspace = true
+const-str.workspace = true
 crc.workspace = true
 windmill-macros.workspace = true
 windmill-parser-sql.workspace = true

backend/windmill-common/src/ai_providers.rs

@@ -11,6 +11,7 @@ lazy_static::lazy_static! {
 }
 
 pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1";
+pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
 
 #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)]
 #[serde(rename_all = "lowercase")]
@@ -38,6 +39,11 @@ impl AIProvider {
         region: Option<String>,
         db: &DB,
     ) -> Result<String> {
+        // If a base URL is provided in the resource, use it
+        if let Some(base_url) = resource_base_url {
+            return Ok(base_url);
+        }
+
         match self {
             AIProvider::OpenAI => {
                 // Check for Azure base path override
@@ -62,28 +68,20 @@ impl AIProvider {
                 Ok(azure_base_path.unwrap_or("https://api.openai.com/v1".to_string()))
             }
             AIProvider::DeepSeek => Ok("https://api.deepseek.com/v1".to_string()),
-            AIProvider::GoogleAI => {
-                Ok("https://generativelanguage.googleapis.com/v1beta/openai".to_string())
-            }
+            AIProvider::GoogleAI => Ok(GOOGLE_AI_BASE_URL.to_string()),
             AIProvider::Groq => Ok("https://api.groq.com/openai/v1".to_string()),
             AIProvider::OpenRouter => Ok("https://openrouter.ai/api/v1".to_string()),
             AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()),
             AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()),
             AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()),
-            p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI) => {
-                if let Some(base_url) = resource_base_url {
-                    Ok(base_url)
-                } else {
-                    Err(Error::BadRequest(format!(
-                        "{:?} provider requires a base URL in the resource",
-                        p
-                    )))
-                }
-            }
             AIProvider::AWSBedrock => Ok(format!(
                 "https://bedrock-runtime.{}.amazonaws.com",
                 region.unwrap_or_else(|| "us-east-1".to_string())
             )),
+            AIProvider::CustomAI | AIProvider::AzureOpenAI => Err(Error::BadRequest(format!(
+                "{:?} provider requires a base URL in the resource",
+                self
+            ))),
         }
     }
 

backend/windmill-common/src/db.rs

@@ -148,6 +148,13 @@ impl<'a, T: Authable + Sync> DbWithOptAuthed<'a, T> {
             DbWithOptAuthed::DB { .. } => None,
         }
     }
+
+    pub fn audit_author(&self) -> Option<&AuditAuthor> {
+        match self {
+            DbWithOptAuthed::UserDB { .. } => None,
+            DbWithOptAuthed::DB { audit_author, .. } => Some(audit_author),
+        }
+    }
 }
 
 impl<'c, 'd, T: Authable + Sync> Acquire<'c> for &'c DbWithOptAuthed<'d, T> {

backend/windmill-common/src/global_settings.rs

@@ -46,7 +46,9 @@ pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
 pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
 pub const OTEL_SETTING: &str = "otel";
+pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy";
 pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
+pub const SECRET_BACKEND_SETTING: &str = "secret_backend";
 
 pub const ENV_SETTINGS: &[&str] = &[
     "DISABLE_NSJAIL",

backend/windmill-common/src/jobs.rs

@@ -825,6 +825,9 @@ lazy_static::lazy_static! {
     pub static ref TAGS_ARE_SENSITIVE: bool = std::env::var("TAGS_ARE_SENSITIVE").map(
         |v| v.parse().unwrap()
     ).unwrap_or(false);
+    pub static ref HIDE_WORKERS_FOR_NON_ADMINS: bool = std::env::var("HIDE_WORKERS_FOR_NON_ADMINS").map(
+        |v| v.parse().unwrap()
+    ).unwrap_or(false);
 }
 
 pub async fn check_tag_available_for_workspace_internal(

backend/windmill-common/src/lib.rs

@@ -55,6 +55,7 @@ pub mod indexer;
 pub mod job_metrics;
 #[cfg(all(feature = "parquet", feature = "private"))]
 pub mod job_s3_helpers_ee;
+pub mod min_version;
 #[cfg(feature = "parquet")]
 pub mod job_s3_helpers_oss;
 pub mod workspace_dependencies;
@@ -78,6 +79,7 @@ pub mod result_stream;
 pub mod runnable_settings;
 pub mod s3_helpers;
 pub mod schedule;
+pub mod secret_backend;
 pub mod schema;
 pub mod scripts;
 pub mod server;

backend/windmill-common/src/min_version.rs

@@ -0,0 +1,175 @@
+use crate::error::{self, Error};
+use semver::Version;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+// ============ Feature Definitions ============
+
+pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing");
+pub const MIN_VERSION_SUPPORTS_DEBOUNCING_V2: VC = vc(1, 597, 0, "Debouncing V2");
+pub const MIN_VERSION_IS_AT_LEAST_1_595: VC = vc(1, 595, 0, "Flow status separate table");
+pub const MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0: VC = vc(1, 592, 0, "Runnable settings V0");
+pub const MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES: VC = vc(1, 587, 0, "Workspace dependencies");
+pub const MIN_VERSION_SUPPORTS_DEBOUNCING: VC = vc(1, 566, 0, "Debouncing");
+pub const MIN_VERSION_IS_AT_LEAST_1_461: VC = vc(1, 461, 0, "V2 job tables");
+pub const MIN_VERSION_IS_AT_LEAST_1_440: VC = vc(1, 440, 0, "Flow node value on pull");
+pub const MIN_VERSION_IS_AT_LEAST_1_432: VC = vc(1, 432, 0, "Flow script job kind");
+pub const MIN_VERSION_IS_AT_LEAST_1_427: VC = vc(1, 427, 0, "Flow version lite table");
+
+// TODO: Currently only shows warning in frontend. In the future,
+// workers below this version should be terminated automatically.
+
+/// Minimum version workers must have to stay connected.
+/// Served via: GET /api/settings/min_keep_alive_version
+/// Also used by vc() for compile-time checks.
+pub const MIN_KEEP_ALIVE_VERSION: (u64, u64, u64) = (1, 400, 0);
+
+// Compile-time check: must lag at least 50 minor versions behind current.
+// NOTE: The 50 version lag is a constant and should NEVER be changed. If this check
+// fails, wait until enough versions have passed rather than reducing the lag requirement.
+// Skip check if GIT_VERSION is "unknown-version" (no git tags available during build)
+const _: () = assert!(
+    !const_str::contains!(crate::utils::GIT_VERSION, ".") ||
+    const_str::parse!(const_str::split!(crate::utils::GIT_VERSION, ".")[1], u64) - MIN_KEEP_ALIVE_VERSION.1 >= 50
+);
+
+// ============ Implementation ============
+lazy_static::lazy_static! {
+    // Global minimum version across all workers (for feature flags)
+    pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
+}
+
+/// Creates a VersionConstraint with compile-time assertion that version > MIN_KEEP_ALIVE_VERSION.
+/// When MIN_KEEP_ALIVE_VERSION is raised, obsolete constraints will fail to compile.
+pub const fn vc(major: u64, minor: u64, patch: u64, name: &'static str) -> VersionConstraint {
+    let is_greater = major > MIN_KEEP_ALIVE_VERSION.0
+        || (major == MIN_KEEP_ALIVE_VERSION.0 && minor > MIN_KEEP_ALIVE_VERSION.1)
+        || (major == MIN_KEEP_ALIVE_VERSION.0 && minor == MIN_KEEP_ALIVE_VERSION.1 && patch > MIN_KEEP_ALIVE_VERSION.2);
+    assert!(
+        is_greater,
+        "Feature version must be > MIN_KEEP_ALIVE_VERSION. Remove this obsolete constraint."
+    );
+    VersionConstraint { available_since: Version::new(major, minor, patch), name }
+}
+
+pub type VC = VersionConstraint;
+
+#[derive(Clone)]
+pub struct VersionConstraint {
+    available_since: Version,
+    name: &'static str,
+}
+
+impl VersionConstraint {
+    pub fn version(&self) -> &Version {
+        &self.available_since
+    }
+
+    pub async fn met(&self) -> bool {
+        let min = MIN_VERSION.read().await;
+        // If MIN_VERSION is 0.0.0, it hasn't been set yet - assume met
+        if *min == Version::new(0, 0, 0) {
+            tracing::warn!("MIN_VERSION not set yet, assuming feature '{}' is met", self.name);
+            return true;
+        }
+        &self.available_since <= &*min
+    }
+
+    pub async fn assert(&self) -> error::Result<()> {
+        if self.met().await {
+            Ok(())
+        } else {
+            Err(Error::WorkersAreBehind {
+                feature: self.name.to_string(),
+                min_version: self.available_since.to_string(),
+            })
+        }
+    }
+}
+
+
+// ============ Version Management ============
+
+use crate::worker::Connection;
+use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
+
+/// Fetches the minimum version across all workers.
+// TODO: consider using HTTP for everything instead of Connection enum
+pub async fn get_min_version(conn: &Connection) -> error::Result<Version> {
+    let fetched = match conn {
+        Connection::Sql(pool) => {
+            let pings = sqlx::query_scalar!(
+                "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
+                GIT_VERSION
+            ).fetch_all(pool).await?;
+
+            pings
+                .iter()
+                .filter(|x| !x.is_empty())
+                .filter_map(|x| {
+                    Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()
+                })
+                .min()
+        }
+        Connection::Http(client) => {
+            Some(
+                client
+                    .get::<String>("/api/agent_workers/get_min_version")
+                    .await
+                    .map(|v| Version::parse(&v))??,
+            )
+        }
+    };
+
+    Ok(fetched.unwrap_or_else(|| GIT_SEM_VERSION.clone()))
+}
+
+/// Updates MIN_VERSION and optionally checks min keep-alive version for workers.
+/// If `_worker_mode` is true, fetches min keep-alive version from server and sends alerts for each worker.
+/// If `initial_load` is true, skips the HTTP fetch to min_keep_alive_version endpoint (server may not be ready).
+pub async fn update_min_version(conn: &Connection, _worker_mode: bool, _worker_names: Vec<String>, _initial_load: bool) {
+    // Update MIN_VERSION
+    match get_min_version(conn).await {
+        Ok(ref mut v) => {
+            let cur = GIT_SEM_VERSION.clone();
+            if v != &cur {
+                tracing::info!("Minimal worker version: {v}");
+            }
+            v.pre = semver::Prerelease::EMPTY;
+            v.build = semver::BuildMetadata::EMPTY;
+            *MIN_VERSION.write().await = v.clone();
+        }
+        Err(e) => tracing::error!("Failed to fetch min version: {:#?}", e),
+    }
+
+    // Workers fetch min keep-alive version from server and send alerts
+    // Skip on initial_load since the server may not be ready yet
+    #[cfg(all(feature = "enterprise", feature = "private"))]
+    if _worker_mode && !_initial_load {
+        if let Connection::Sql(db) = conn {
+            let url = format!("{}/api/min_keep_alive_version", *crate::BASE_INTERNAL_URL);
+            match crate::utils::HTTP_CLIENT.get(&url).send().await {
+                Ok(resp) => match resp.text().await {
+                    Ok(v) => match Version::parse(&v) {
+                        Ok(min_keep_alive) => {
+                            let current = GIT_SEM_VERSION.clone();
+                            for worker_name in &_worker_names {
+                                crate::ee::simple_alert_helper(
+                                    format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade recommended."),
+                                    format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."),
+                                    &format!("worker-below-min-keep-alive-{worker_name}"),
+                                    || current < min_keep_alive,
+                                    Some("admins"),
+                                    db,
+                                ).await;
+                            }
+                        }
+                        Err(e) => tracing::error!("Failed to parse min keep-alive version: {:#?}", e),
+                    },
+                    Err(e) => tracing::error!("Failed to read min keep-alive version response: {:#?}", e),
+                },
+                Err(e) => tracing::error!("Failed to fetch min keep-alive version: {:#?}", e),
+            }
+        }
+    }
+}

backend/windmill-common/src/runnable_settings/mod.rs

@@ -9,19 +9,16 @@ use std::{
 use serde::{de::DeserializeOwned, Serialize};
 use sqlx::{postgres::PgRow, FromRow, Pool, Postgres};
 
-use crate::{error, make_static};
+use crate::{error, make_static, min_version::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0};
 
 lazy_static::lazy_static! {
     static ref WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: bool = std::env::var("WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0").is_ok();
-    pub static ref MIN_VERSION_RUNNABLE_SETTINGS_V0: semver::Version = semver::Version::new(1, 592, 0);
 }
 
 pub async fn min_version_supports_runnable_settings_v0() -> bool {
-    // Check if workers support workspace dependencies feature
+    // Check if workers support runnable settings feature
     if !*WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0
-        && !*crate::worker::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
-            .read()
-            .await
+        && !MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0.met().await
     {
         tracing::debug!(
             "Internal: min version does not support runnable settings v0, falling back to old system",

backend/windmill-common/src/s3_helpers.rs

@@ -51,6 +51,88 @@ use tokio::task;
 #[cfg(feature = "parquet")]
 use windmill_parser_sql::S3ModeFormat;
 
+use std::collections::HashMap;
+
+lazy_static::lazy_static! {
+    static ref S3_BUCKET_RESTRICTIONS: Option<HashMap<String, Vec<String>>> = {
+        parse_bucket_restrictions()
+    };
+}
+
+/// Parses the S3_BUCKETS_WORKSPACE_RESTRICTIONS environment variable
+/// Format: bucket_a:workspace1,workspace2;bucket_b:workspace3,workspace4
+fn parse_bucket_restrictions() -> Option<HashMap<String, Vec<String>>> {
+    let env_var = std::env::var("S3_BUCKETS_WORKSPACE_RESTRICTIONS").ok()?;
+
+    if env_var.trim().is_empty() {
+        return None;
+    }
+
+    let mut restrictions = HashMap::new();
+
+    for bucket_rule in env_var.split(';') {
+        let bucket_rule = bucket_rule.trim();
+        if bucket_rule.is_empty() {
+            continue;
+        }
+
+        let parts: Vec<&str> = bucket_rule.splitn(2, ':').collect();
+        if parts.len() != 2 {
+            tracing::warn!(
+                "Invalid bucket restriction format: '{}'. Expected 'bucket:workspace1,workspace2'",
+                bucket_rule
+            );
+            continue;
+        }
+
+        let bucket_name = parts[0].trim().to_string();
+        let workspaces: Vec<String> = parts[1]
+            .split(',')
+            .map(|w| w.trim().to_string())
+            .filter(|w| !w.is_empty())
+            .collect();
+
+        if workspaces.is_empty() {
+            tracing::warn!(
+                "No workspaces specified for bucket '{}', skipping restriction",
+                bucket_name
+            );
+            continue;
+        }
+
+        restrictions.insert(bucket_name, workspaces);
+    }
+
+    if restrictions.is_empty() {
+        None
+    } else {
+        tracing::info!(
+            "S3 bucket restrictions loaded for {} buckets",
+            restrictions.len()
+        );
+        Some(restrictions)
+    }
+}
+
+/// Checks if a workspace is allowed to access a given bucket based on restrictions
+/// Returns Ok(()) if access is allowed, Err if restricted
+pub fn check_bucket_workspace_restriction(
+    bucket_name: &str,
+    workspace_id: &str,
+) -> error::Result<()> {
+    if let Some(ref restrictions) = *S3_BUCKET_RESTRICTIONS {
+        if let Some(allowed_workspaces) = restrictions.get(bucket_name) {
+            if !allowed_workspaces.contains(&workspace_id.to_string()) {
+                return Err(error::Error::NotAuthorized(format!(
+                    "Workspace '{}' is not authorized to access bucket '{}'",
+                    workspace_id, bucket_name
+                )));
+            }
+        }
+    }
+    Ok(())
+}
+
 #[cfg(feature = "parquet")]
 #[derive(Clone)]
 pub struct ExpirableObjectStore {

backend/windmill-common/src/secret_backend/database.rs

@@ -0,0 +1,126 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+//! Database backend for secret storage
+//!
+//! This is the default backend that stores secrets encrypted in the PostgreSQL database
+//! using the existing magic_crypt encryption with workspace-specific keys.
+
+use async_trait::async_trait;
+
+use crate::db::DB;
+use crate::error::{Error, Result};
+use crate::variables::{build_crypt, decrypt, encrypt};
+
+use super::SecretBackend;
+
+/// Database-backed secret storage
+///
+/// This backend stores secrets encrypted in the `variable` table using
+/// the workspace's encryption key. This is the default and original
+/// behavior of Windmill.
+pub struct DatabaseBackend {
+    db: DB,
+}
+
+impl DatabaseBackend {
+    /// Create a new database backend
+    pub fn new(db: DB) -> Self {
+        Self { db }
+    }
+}
+
+#[async_trait]
+impl SecretBackend for DatabaseBackend {
+    async fn get_secret(&self, workspace_id: &str, path: &str) -> Result<String> {
+        let variable = sqlx::query!(
+            "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true",
+            path,
+            workspace_id
+        )
+        .fetch_optional(&self.db)
+        .await?;
+
+        let variable = variable.ok_or_else(|| {
+            Error::NotFound(format!(
+                "Secret variable {} not found in workspace {}",
+                path, workspace_id
+            ))
+        })?;
+
+        let value = variable.value;
+        if value.is_empty() {
+            return Ok(String::new());
+        }
+
+        let mc = build_crypt(&self.db, workspace_id).await?;
+        decrypt(&mc, value).map_err(|e| {
+            Error::internal_err(format!("Error decrypting variable {}: {}", path, e))
+        })
+    }
+
+    async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> {
+        let mc = build_crypt(&self.db, workspace_id).await?;
+        let encrypted_value = encrypt(&mc, value);
+
+        // Update the value in the database
+        // Note: This assumes the variable row already exists (created via the normal API)
+        let result = sqlx::query!(
+            "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true",
+            encrypted_value,
+            path,
+            workspace_id
+        )
+        .execute(&self.db)
+        .await?;
+
+        if result.rows_affected() == 0 {
+            return Err(Error::NotFound(format!(
+                "Secret variable {} not found in workspace {}",
+                path, workspace_id
+            )));
+        }
+
+        Ok(())
+    }
+
+    async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> {
+        // For database backend, deletion is handled by the normal variable deletion flow
+        // The encrypted value is just deleted along with the row
+        // This method is a no-op for database backend since the caller handles the DELETE
+        Ok(())
+    }
+
+    fn backend_name(&self) -> &'static str {
+        "database"
+    }
+}
+
+/// Encrypt a value for storage in the database
+///
+/// This is a convenience function for use when creating new secrets.
+pub async fn encrypt_for_database(db: &DB, workspace_id: &str, value: &str) -> Result<String> {
+    let mc = build_crypt(db, workspace_id).await?;
+    Ok(encrypt(&mc, value))
+}
+
+/// Decrypt a value from the database
+///
+/// This is a convenience function for reading secrets.
+pub async fn decrypt_from_database(
+    db: &DB,
+    workspace_id: &str,
+    encrypted_value: String,
+) -> Result<String> {
+    if encrypted_value.is_empty() {
+        return Ok(String::new());
+    }
+    let mc = build_crypt(db, workspace_id).await?;
+    decrypt(&mc, encrypted_value)
+        .map_err(|e| Error::internal_err(format!("Error decrypting value: {}", e)))
+}

backend/windmill-common/src/secret_backend/mod.rs

@@ -0,0 +1,133 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+//! Secret Backend abstraction for storing secrets in external vaults
+//!
+//! This module provides a trait-based abstraction for secret storage,
+//! allowing secrets to be stored in the database (default) or in external
+//! vaults like HashiCorp Vault (Enterprise Edition).
+
+pub mod database;
+
+#[cfg(feature = "private")]
+pub mod vault_ee;
+
+pub mod vault_oss;
+
+#[cfg(test)]
+mod tests;
+
+#[cfg(feature = "private")]
+pub use vault_ee::*;
+
+#[cfg(not(feature = "private"))]
+pub use vault_oss::*;
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+
+use crate::error::Result;
+
+/// Trait for secret storage backends
+///
+/// Implementations of this trait handle the storage and retrieval of secrets.
+/// The default implementation stores secrets encrypted in the database.
+/// Enterprise Edition supports HashiCorp Vault as an alternative backend.
+#[async_trait]
+pub trait SecretBackend: Send + Sync {
+    /// Retrieve a secret value
+    ///
+    /// # Arguments
+    /// * `workspace_id` - The workspace identifier
+    /// * `path` - The path/name of the secret variable
+    ///
+    /// # Returns
+    /// The decrypted secret value
+    async fn get_secret(&self, workspace_id: &str, path: &str) -> Result<String>;
+
+    /// Store a secret value
+    ///
+    /// # Arguments
+    /// * `workspace_id` - The workspace identifier
+    /// * `path` - The path/name of the secret variable
+    /// * `value` - The plaintext secret value to store
+    async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()>;
+
+    /// Delete a secret
+    ///
+    /// # Arguments
+    /// * `workspace_id` - The workspace identifier
+    /// * `path` - The path/name of the secret variable
+    async fn delete_secret(&self, workspace_id: &str, path: &str) -> Result<()>;
+
+    /// Get the name of this backend for logging/debugging
+    fn backend_name(&self) -> &'static str;
+}
+
+/// Configuration for secret storage backend
+///
+/// This enum is stored in global_settings and determines which backend
+/// is used for secret storage at the instance level.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(tag = "type")]
+pub enum SecretBackendConfig {
+    /// Store secrets encrypted in the database (default behavior)
+    Database,
+    /// Store secrets in HashiCorp Vault (Enterprise Edition only)
+    HashiCorpVault(VaultSettings),
+}
+
+impl Default for SecretBackendConfig {
+    fn default() -> Self {
+        SecretBackendConfig::Database
+    }
+}
+
+/// Settings for HashiCorp Vault integration
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct VaultSettings {
+    /// Vault server address (e.g., "https://vault.company.com:8200")
+    pub address: String,
+    /// KV v2 mount path (e.g., "windmill")
+    pub mount_path: String,
+    /// JWT auth role name configured in Vault (used for JWT/OIDC auth)
+    /// Optional - if not provided, token auth is used
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub jwt_role: Option<String>,
+    /// Vault Enterprise namespace (optional)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub namespace: Option<String>,
+    /// Static Vault token for testing/development (optional)
+    /// If provided, this is used instead of JWT authentication
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub token: Option<String>,
+}
+
+/// Result of a secret migration operation
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SecretMigrationReport {
+    /// Total number of secrets found
+    pub total_secrets: usize,
+    /// Number of secrets successfully migrated
+    pub migrated_count: usize,
+    /// Number of secrets that failed to migrate
+    pub failed_count: usize,
+    /// Details of any failures
+    pub failures: Vec<SecretMigrationFailure>,
+}
+
+/// Details of a failed secret migration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SecretMigrationFailure {
+    /// Workspace ID where the secret is located
+    pub workspace_id: String,
+    /// Path of the secret that failed to migrate
+    pub path: String,
+    /// Error message
+    pub error: String,
+}

backend/windmill-common/src/secret_backend/tests.rs

@@ -0,0 +1,143 @@
+/*
+ * Integration tests for secret backend
+ *
+ * These tests require a running Vault instance at http://127.0.0.1:8200
+ * with the token "test-root-token" and a KV v2 mount at "windmill".
+ *
+ * Run Vault dev server:
+ * podman run -d --name vault-test --rm -p 8200:8200 \
+ *   -e 'VAULT_DEV_ROOT_TOKEN_ID=test-root-token' \
+ *   -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \
+ *   docker.io/hashicorp/vault:latest
+ *
+ * Then enable the windmill mount:
+ * curl --header "X-Vault-Token: test-root-token" \
+ *   --request POST \
+ *   --data '{"type": "kv", "options": {"version": "2"}}' \
+ *   http://127.0.0.1:8200/v1/sys/mounts/windmill
+ */
+
+#[cfg(test)]
+mod tests {
+    use crate::secret_backend::{SecretBackend, VaultBackend, VaultSettings};
+
+    fn test_settings() -> VaultSettings {
+        VaultSettings {
+            address: "http://127.0.0.1:8200".to_string(),
+            mount_path: "windmill".to_string(),
+            jwt_role: Some("windmill-secrets".to_string()),
+            namespace: None,
+            token: Some("test-root-token".to_string()),
+        }
+    }
+
+    #[tokio::test]
+    #[ignore] // Run with --ignored to execute
+    async fn test_vault_write_and_read() {
+        let settings = test_settings();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-ws-1";
+        let path = "test-secret";
+        let value = "my-super-secret-value";
+
+        // Write secret
+        backend
+            .set_secret(workspace_id, path, value)
+            .await
+            .expect("Failed to write secret");
+
+        // Read secret
+        let read_value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read secret");
+
+        assert_eq!(read_value, value);
+
+        // Delete secret
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete secret");
+
+        // Verify deleted
+        let result = backend.get_secret(workspace_id, path).await;
+        assert!(result.is_err(), "Secret should be deleted");
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_vault_multiple_secrets() {
+        let settings = test_settings();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-ws-2";
+
+        // Write multiple secrets
+        for i in 0..5 {
+            let path = format!("secret-{}", i);
+            let value = format!("value-{}", i);
+            backend
+                .set_secret(workspace_id, &path, &value)
+                .await
+                .expect(&format!("Failed to write secret-{}", i));
+        }
+
+        // Read and verify
+        for i in 0..5 {
+            let path = format!("secret-{}", i);
+            let expected = format!("value-{}", i);
+            let value = backend
+                .get_secret(workspace_id, &path)
+                .await
+                .expect(&format!("Failed to read secret-{}", i));
+            assert_eq!(value, expected);
+        }
+
+        // Cleanup
+        for i in 0..5 {
+            let path = format!("secret-{}", i);
+            backend
+                .delete_secret(workspace_id, &path)
+                .await
+                .expect(&format!("Failed to delete secret-{}", i));
+        }
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_vault_overwrite_secret() {
+        let settings = test_settings();
+        let backend = VaultBackend::new(settings);
+
+        let workspace_id = "test-ws-3";
+        let path = "overwrite-test";
+
+        // Write initial value
+        backend
+            .set_secret(workspace_id, path, "initial-value")
+            .await
+            .expect("Failed to write initial value");
+
+        // Overwrite
+        backend
+            .set_secret(workspace_id, path, "new-value")
+            .await
+            .expect("Failed to overwrite");
+
+        // Read and verify
+        let value = backend
+            .get_secret(workspace_id, path)
+            .await
+            .expect("Failed to read");
+
+        assert_eq!(value, "new-value");
+
+        // Cleanup
+        backend
+            .delete_secret(workspace_id, path)
+            .await
+            .expect("Failed to delete");
+    }
+}

backend/windmill-common/src/secret_backend/vault_oss.rs

@@ -0,0 +1,107 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2024
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+//! HashiCorp Vault secret backend stubs (Open Source Edition)
+//!
+//! This module provides stub implementations for Vault integration.
+//! The actual Vault integration requires Enterprise Edition.
+
+use std::sync::Arc;
+
+use crate::db::DB;
+use crate::error::{Error, Result};
+
+use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings};
+
+/// Stub VaultBackend for OSS - all operations return EE required error
+pub struct VaultBackend;
+
+impl VaultBackend {
+    pub fn new(_settings: VaultSettings) -> Self {
+        Self
+    }
+}
+
+#[async_trait::async_trait]
+impl SecretBackend for VaultBackend {
+    async fn get_secret(&self, _workspace_id: &str, _path: &str) -> Result<String> {
+        Err(Error::internal_err(
+            "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+        ))
+    }
+
+    async fn set_secret(&self, _workspace_id: &str, _path: &str, _value: &str) -> Result<()> {
+        Err(Error::internal_err(
+            "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+        ))
+    }
+
+    async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> {
+        Err(Error::internal_err(
+            "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+        ))
+    }
+
+    fn backend_name(&self) -> &'static str {
+        "hashicorp_vault"
+    }
+}
+
+/// Create the appropriate secret backend based on configuration
+///
+/// In OSS, always returns DatabaseBackend regardless of config.
+/// Vault configuration is ignored with a warning.
+pub async fn create_secret_backend(
+    db: DB,
+    config: &SecretBackendConfig,
+) -> Result<Arc<dyn SecretBackend>> {
+    match config {
+        SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db))),
+        SecretBackendConfig::HashiCorpVault(_) => {
+            tracing::warn!(
+                "HashiCorp Vault is configured but requires Enterprise Edition. \
+                 Falling back to database backend."
+            );
+            Ok(Arc::new(DatabaseBackend::new(db)))
+        }
+    }
+}
+
+/// Test connection to Vault (OSS stub)
+pub async fn test_vault_connection(_settings: &VaultSettings, _db: Option<&DB>) -> Result<()> {
+    Err(Error::internal_err(
+        "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+    ))
+}
+
+/// Migrate secrets from database to Vault (OSS stub)
+pub async fn migrate_secrets_to_vault(
+    _db: &DB,
+    _settings: &VaultSettings,
+) -> Result<SecretMigrationReport> {
+    Err(Error::internal_err(
+        "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+    ))
+}
+
+/// Migrate secrets from Vault back to database (OSS stub)
+pub async fn migrate_secrets_to_database(
+    _db: &DB,
+    _settings: &VaultSettings,
+) -> Result<SecretMigrationReport> {
+    Err(Error::internal_err(
+        "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+    ))
+}
+
+/// Generate a JWT for Vault authentication (OSS stub)
+pub async fn generate_vault_jwt(_db: &DB, _vault_address: &str) -> Result<String> {
+    Err(Error::internal_err(
+        "HashiCorp Vault integration requires Enterprise Edition".to_string(),
+    ))
+}

backend/windmill-common/src/worker.rs

@@ -5,7 +5,6 @@ use const_format::concatcp;
 use itertools::Itertools;
 use regex::Regex;
 use reqwest_middleware::ClientWithMiddleware;
-use semver::Version;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use serde_json::{json, value::RawValue};
 use sqlx::{types::Json, Pool, Postgres};
@@ -34,7 +33,7 @@ use crate::{
     global_settings::CUSTOM_TAGS_SETTING,
     indexer::TantivyIndexerSettings,
     server::Smtp,
-    utils::{merge_nested_raw_values_to_array, merge_raw_values_to_array, GIT_SEM_VERSION},
+    utils::{merge_nested_raw_values_to_array, merge_raw_values_to_array},
     KillpillSender, BASE_INTERNAL_URL, DB,
 };
 
@@ -264,24 +263,6 @@ lazy_static::lazy_static! {
     .and_then(|x| x.parse::<bool>().ok())
     .unwrap_or(false);
 
-    pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
-    pub static ref MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_SUPPORTS_DEBOUNCING_V2: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    /// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0)
-    /// This flag is updated during worker initialization by checking the minimum version across all workers
-    /// When false, creation of workspace dependencies is forbidden and extraction of external workspace dependencies will error
-    pub static ref MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    /// Global flag indicating if all workers support the debouncing feature (>= 1.566.0)
-    /// Debouncing consolidates multiple dependency job requests within a time window to avoid redundant work
-    /// This flag is updated during worker initialization by checking the minimum version across all workers
-    pub static ref MIN_VERSION_SUPPORTS_DEBOUNCING: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_IS_AT_LEAST_1_461: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_IS_AT_LEAST_1_427: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_IS_AT_LEAST_1_432: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_IS_AT_LEAST_1_440: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-    pub static ref MIN_VERSION_IS_AT_LEAST_1_595: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
-
     // Features flags:
     pub static ref DISABLE_FLOW_SCRIPT: bool = std::env::var("DISABLE_FLOW_SCRIPT").ok().is_some_and(|x| x == "1" || x == "true");
 
@@ -1242,82 +1223,6 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
     }
 }
 
-pub async fn get_min_version(conn: &Connection) -> error::Result<Version> {
-    use crate::utils::GIT_VERSION;
-
-    let fetched = match conn {
-        Connection::Sql(pool) => {
-            // fetch all pings with a different version than self from the last 5 minutes.
-            let pings = sqlx::query_scalar!(
-                "SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
-                GIT_VERSION
-            ).fetch_all(pool).await?;
-
-            pings
-                .iter()
-                .filter(|x| !x.is_empty())
-                .filter_map(|x| {
-                    semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()
-                })
-                .min()
-        }
-        Connection::Http(client) => {
-            // Fetch min version from server
-            Some(
-                client
-                    .get::<String>("/api/agent_workers/get_min_version")
-                    .await
-                    .map(|v| Version::parse(&v))??,
-            )
-        }
-    };
-
-    Ok(fetched.unwrap_or_else(|| GIT_SEM_VERSION.clone()))
-}
-
-pub async fn update_min_version(conn: &Connection) -> bool {
-    tracing::debug!("Updating min version");
-    use crate::utils::GIT_SEM_VERSION;
-
-    let cur_version = GIT_SEM_VERSION.clone();
-
-    let min_version = match get_min_version(conn).await {
-        Ok(v) => v,
-        Err(e) => {
-            tracing::error!(
-                "Failed to fetch min version: {:#?}, using current version",
-                e
-            );
-            cur_version.clone()
-        }
-    };
-
-    if min_version != cur_version {
-        tracing::info!("Minimal worker version: {min_version}");
-    }
-
-    *MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING.write().await =
-        min_version >= Version::new(1, 602, 0);
-    *MIN_VERSION_SUPPORTS_DEBOUNCING_V2.write().await = min_version >= Version::new(1, 597, 0);
-    *MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0.write().await =
-        min_version >= *crate::runnable_settings::MIN_VERSION_RUNNABLE_SETTINGS_V0;
-
-    // Workspace dependencies feature requires minimum version across all workers
-    *MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES.write().await = min_version
-        >= Version::parse(crate::workspace_dependencies::MIN_VERSION_WORKSPACE_DEPENDENCIES)
-            .unwrap();
-    // Debouncing feature requires minimum version 1.566.0 across all workers
-    // This ensures all workers can handle debounce keys and stale data accumulation
-    *MIN_VERSION_SUPPORTS_DEBOUNCING.write().await = min_version >= Version::new(1, 566, 0);
-    *MIN_VERSION_IS_AT_LEAST_1_461.write().await = min_version >= Version::new(1, 461, 0);
-    *MIN_VERSION_IS_AT_LEAST_1_427.write().await = min_version >= Version::new(1, 427, 0);
-    *MIN_VERSION_IS_AT_LEAST_1_432.write().await = min_version >= Version::new(1, 432, 0);
-    *MIN_VERSION_IS_AT_LEAST_1_440.write().await = min_version >= Version::new(1, 440, 0);
-    *MIN_VERSION_IS_AT_LEAST_1_595.write().await = min_version >= Version::new(1, 595, 0);
-
-    *MIN_VERSION.write().await = min_version.clone();
-    min_version >= cur_version
-}
 
 #[derive(Serialize, Deserialize)]
 pub enum PingType {

backend/windmill-common/src/workspace_dependencies.rs

@@ -9,6 +9,7 @@ use crate::{
         set_cached_is_unnamed_workspace_dependencies_exists,
     },
     error,
+    min_version::MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES,
     scripts::ScriptLang,
     utils::calculate_hash,
     worker::Connection,
@@ -31,9 +32,7 @@ pub const MIN_VERSION_WORKSPACE_DEPENDENCIES: &str = "1.587.0";
 pub async fn min_version_supports_v0_workspace_dependencies() -> error::Result<()> {
     // Check if workers support workspace dependencies feature
     if !*WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES
-        && !*crate::worker::MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES
-            .read()
-            .await
+        && !MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES.met().await
     {
         tracing::warn!(
             "Workspace dependencies feature will be disabled because not all workers support it (minimum version {} required)",

backend/windmill-local/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "windmill-local"
+version = "0.1.0"
+edition = "2021"
+description = "Windmill local mode with libSQL/Turso support for preview execution"
+
+[dependencies]
+# libSQL - Turso's SQLite fork (used for local and remote Turso connections)
+# Note: libsql IS the Turso database driver - Turso is built on libSQL
+libsql = "0.9"
+
+# Async runtime
+tokio = { version = "1", features = ["full", "sync", "macros"] }
+
+# Serialization
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+# Error handling
+anyhow = "1"
+thiserror = "1"
+
+# UUID for job IDs
+uuid = { version = "1", features = ["v4", "serde"] }
+
+# Timestamps
+chrono = { version = "0.4", features = ["serde"] }
+
+# Tracing
+tracing = "0.1"
+
+# HTTP server
+axum = "0.7"
+tower = "0.4"
+tower-http = { version = "0.5", features = ["cors", "trace"] }
+
+# Windmill types (flows, scripts, etc.)
+windmill-common = { path = "../windmill-common", default-features = false }
+
+# For input transforms (JavaScript evaluation)
+rquickjs = { version = "0.8", features = ["bindgen", "classes", "loader", "array-buffer", "futures"] }
+
+# For recursive async functions
+async-recursion = "1"
+
+[dev-dependencies]
+tokio-test = "0.4"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+[[example]]
+name = "local_server"
+path = "examples/local_server.rs"

backend/windmill-local/examples/local_server.rs

@@ -0,0 +1,65 @@
+//! Example: Run the Windmill local server
+//!
+//! This demonstrates running a local Windmill server with libSQL/SQLite backend.
+//!
+//! Run with:
+//!     cargo run -p windmill-local --example local_server
+//!
+//! Test with:
+//!     # Health check
+//!     curl http://localhost:8000/health
+//!
+//!     # Run a bash preview (async)
+//!     curl -X POST http://localhost:8000/api/w/local/jobs/run/preview \
+//!       -H "Content-Type: application/json" \
+//!       -d '{"content": "echo Hello World", "language": "bash"}'
+//!
+//!     # Run a bash preview and wait for result
+//!     curl -X POST http://localhost:8000/api/w/local/jobs/run_wait_result/preview \
+//!       -H "Content-Type: application/json" \
+//!       -d '{"content": "echo 42", "language": "bash"}'
+//!
+//!     # Run a Python preview
+//!     curl -X POST http://localhost:8000/api/w/local/jobs/run_wait_result/preview \
+//!       -H "Content-Type: application/json" \
+//!       -d '{"content": "def main(x=1): return x * 2", "language": "python3", "args": {"x": 21}}'
+//!
+//!     # Run a flow preview (linear flow with two steps)
+//!     curl -X POST http://localhost:8000/api/w/local/jobs/run_wait_result/preview_flow \
+//!       -H "Content-Type: application/json" \
+//!       -d '{
+//!         "value": {
+//!           "modules": [
+//!             {"id": "step1", "value": {"type": "rawscript", "language": "bash", "content": "echo 10"}},
+//!             {"id": "step2", "value": {"type": "identity"}}
+//!           ]
+//!         },
+//!         "args": {}
+//!       }'
+
+use std::net::SocketAddr;
+use windmill_local::LocalServer;
+
+#[tokio::main]
+async fn main() {
+    // Initialize tracing
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::from_default_env()
+                .add_directive("windmill_local=info".parse().unwrap()),
+        )
+        .init();
+
+    let addr: SocketAddr = "0.0.0.0:8000".parse().unwrap();
+    println!("Starting Windmill Local Server on {}", addr);
+    println!();
+    println!("Test endpoints:");
+    println!("  Health:    curl http://localhost:8000/health");
+    println!("  Preview:   curl -X POST http://localhost:8000/api/w/local/jobs/run_wait_result/preview \\");
+    println!("               -H 'Content-Type: application/json' \\");
+    println!("               -d '{{\"content\": \"echo Hello\", \"language\": \"bash\"}}'");
+    println!();
+
+    let server = LocalServer::new(addr).await.expect("Failed to create server");
+    server.run().await.expect("Server error");
+}

backend/windmill-local/src/db.rs

@@ -0,0 +1,161 @@
+//! Database connection and initialization for local mode
+
+use libsql::{Builder, Connection, Database};
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+use crate::error::Result;
+use crate::schema;
+
+/// Local database wrapper
+///
+/// For local mode, we use a single connection with in-process coordination.
+/// This simplifies the implementation since we don't need `FOR UPDATE SKIP LOCKED`.
+pub struct LocalDb {
+    #[allow(dead_code)]
+    db: Database,
+    /// Single connection for all operations (simplifies transaction handling)
+    conn: Arc<Mutex<Connection>>,
+}
+
+impl LocalDb {
+    /// Create an in-memory database (for testing/ephemeral use)
+    pub async fn in_memory() -> Result<Self> {
+        let db = Builder::new_local(":memory:").build().await?;
+        let conn = db.connect()?;
+        let local_db = Self {
+            db,
+            conn: Arc::new(Mutex::new(conn)),
+        };
+        local_db.init_schema().await?;
+        Ok(local_db)
+    }
+
+    /// Create a file-based database
+    pub async fn file(path: &str) -> Result<Self> {
+        let db = Builder::new_local(path).build().await?;
+        let conn = db.connect()?;
+        let local_db = Self {
+            db,
+            conn: Arc::new(Mutex::new(conn)),
+        };
+        local_db.init_schema().await?;
+        Ok(local_db)
+    }
+
+    /// Create a Turso remote database connection
+    /// This would be used for the multi-writer scenario
+    pub async fn turso_remote(url: &str, auth_token: &str) -> Result<Self> {
+        let db = Builder::new_remote(url.to_string(), auth_token.to_string())
+            .build()
+            .await?;
+        let conn = db.connect()?;
+        let local_db = Self {
+            db,
+            conn: Arc::new(Mutex::new(conn)),
+        };
+        local_db.init_schema().await?;
+        Ok(local_db)
+    }
+
+    /// Initialize the schema
+    async fn init_schema(&self) -> Result<()> {
+        let conn = self.conn.lock().await;
+        // Execute schema as multiple statements
+        conn.execute_batch(schema::SCHEMA).await?;
+        Ok(())
+    }
+
+    /// Reset the database (drop and recreate all tables)
+    pub async fn reset(&self) -> Result<()> {
+        let conn = self.conn.lock().await;
+        conn.execute_batch(schema::DROP_SCHEMA).await?;
+        conn.execute_batch(schema::SCHEMA).await?;
+        Ok(())
+    }
+
+    /// Get a reference to the connection (locked)
+    pub async fn conn(&self) -> tokio::sync::MutexGuard<'_, Connection> {
+        self.conn.lock().await
+    }
+
+    /// Execute a simple query that returns no rows
+    pub async fn execute(&self, sql: &str, params: impl libsql::params::IntoParams) -> Result<u64> {
+        let conn = self.conn.lock().await;
+        let rows_affected = conn.execute(sql, params).await?;
+        Ok(rows_affected)
+    }
+
+    /// Execute a query and return all rows
+    pub async fn query(
+        &self,
+        sql: &str,
+        params: impl libsql::params::IntoParams,
+    ) -> Result<libsql::Rows> {
+        let conn = self.conn.lock().await;
+        let rows = conn.query(sql, params).await?;
+        Ok(rows)
+    }
+
+    /// Execute a batch of statements (for transactions)
+    pub async fn execute_batch(&self, sql: &str) -> Result<()> {
+        let conn = self.conn.lock().await;
+        conn.execute_batch(sql).await?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_in_memory_db() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        // Verify tables exist
+        let rows = db
+            .query(
+                "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name",
+                (),
+            )
+            .await
+            .unwrap();
+
+        let mut tables = Vec::new();
+        let mut rows = rows;
+        while let Some(row) = rows.next().await.unwrap() {
+            let name: String = row.get(0).unwrap();
+            tables.push(name);
+        }
+
+        assert!(tables.contains(&"v2_job".to_string()));
+        assert!(tables.contains(&"v2_job_queue".to_string()));
+        assert!(tables.contains(&"v2_job_completed".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_reset_db() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        // Insert a job
+        db.execute(
+            "INSERT INTO v2_job (id, kind) VALUES ('test-uuid', 'preview')",
+            (),
+        )
+        .await
+        .unwrap();
+
+        // Reset
+        db.reset().await.unwrap();
+
+        // Verify job is gone
+        let mut rows = db
+            .query("SELECT COUNT(*) FROM v2_job", ())
+            .await
+            .unwrap();
+        let row = rows.next().await.unwrap().unwrap();
+        let count: i64 = row.get(0).unwrap();
+        assert_eq!(count, 0);
+    }
+}

backend/windmill-local/src/error.rs

@@ -0,0 +1,29 @@
+//! Error types for local mode
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LocalError {
+    #[error("Database error: {0}")]
+    Database(#[from] libsql::Error),
+
+    #[error("Serialization error: {0}")]
+    Serialization(#[from] serde_json::Error),
+
+    #[error("Job not found: {0}")]
+    JobNotFound(uuid::Uuid),
+
+    #[error("Invalid job state: {0}")]
+    InvalidJobState(String),
+
+    #[error("Queue is empty")]
+    QueueEmpty,
+
+    #[error("Execution error: {0}")]
+    Execution(String),
+
+    #[error("Timeout")]
+    Timeout,
+}
+
+pub type Result<T> = std::result::Result<T, LocalError>;

backend/windmill-local/src/executor.rs

@@ -0,0 +1,330 @@
+//! Simple script executor for local mode
+//!
+//! This is a minimal executor that supports a few languages for demonstration.
+//! A full implementation would integrate with windmill-worker's execution logic.
+
+use std::process::Stdio;
+use tokio::process::Command;
+use tokio::io::AsyncReadExt;
+
+use crate::error::{LocalError, Result};
+use crate::jobs::ScriptLang;
+
+/// Result of script execution
+#[derive(Debug)]
+pub struct ExecutionResult {
+    pub success: bool,
+    pub result: serde_json::Value,
+    pub logs: String,
+}
+
+/// Execute a script with the given language and arguments
+pub async fn execute_script(
+    language: ScriptLang,
+    code: &str,
+    args: &serde_json::Value,
+) -> Result<ExecutionResult> {
+    match language {
+        ScriptLang::Bash => execute_bash(code, args).await,
+        ScriptLang::Python3 => execute_python(code, args).await,
+        ScriptLang::Deno => execute_deno(code, args).await,
+        ScriptLang::Bun => execute_bun(code, args).await,
+        _ => Ok(ExecutionResult {
+            success: false,
+            result: serde_json::json!({
+                "error": format!("Language {:?} not supported in local mode yet", language)
+            }),
+            logs: format!("Language {:?} not supported in local mode", language),
+        }),
+    }
+}
+
+/// Execute a bash script
+async fn execute_bash(code: &str, args: &serde_json::Value) -> Result<ExecutionResult> {
+    // Create environment variables from args
+    let mut env_vars = Vec::new();
+    if let serde_json::Value::Object(map) = args {
+        for (key, value) in map {
+            let val_str = match value {
+                serde_json::Value::String(s) => s.clone(),
+                _ => value.to_string(),
+            };
+            env_vars.push((key.to_uppercase(), val_str));
+        }
+    }
+
+    let mut child = Command::new("bash")
+        .arg("-c")
+        .arg(code)
+        .envs(env_vars)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| LocalError::Execution(format!("Failed to spawn bash: {}", e)))?;
+
+    let status = child
+        .wait()
+        .await
+        .map_err(|e| LocalError::Execution(format!("Failed to wait for bash: {}", e)))?;
+
+    let mut stdout = String::new();
+    let mut stderr = String::new();
+
+    if let Some(mut out) = child.stdout.take() {
+        out.read_to_string(&mut stdout).await.ok();
+    }
+    if let Some(mut err) = child.stderr.take() {
+        err.read_to_string(&mut stderr).await.ok();
+    }
+
+    let logs = format!("{}{}", stdout, stderr);
+    let success = status.success();
+
+    // Try to parse stdout as JSON, otherwise use as string
+    let result = if success {
+        let trimmed = stdout.trim();
+        serde_json::from_str(trimmed).unwrap_or_else(|_| serde_json::json!(trimmed))
+    } else {
+        serde_json::json!({ "error": stderr.trim() })
+    };
+
+    Ok(ExecutionResult {
+        success,
+        result,
+        logs,
+    })
+}
+
+/// Execute a Python script
+async fn execute_python(code: &str, args: &serde_json::Value) -> Result<ExecutionResult> {
+    // Wrap the code to handle args and return JSON result
+    let wrapped_code = format!(
+        r#"
+import json
+import sys
+
+# Args passed as JSON
+args = json.loads('''{}''')
+
+# User code
+{}
+
+# Call main if it exists
+if 'main' in dir():
+    result = main(**args)
+    print(json.dumps(result))
+"#,
+        serde_json::to_string(args)?,
+        code
+    );
+
+    let mut child = Command::new("python3")
+        .arg("-c")
+        .arg(&wrapped_code)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| LocalError::Execution(format!("Failed to spawn python3: {}", e)))?;
+
+    let status = child
+        .wait()
+        .await
+        .map_err(|e| LocalError::Execution(format!("Failed to wait for python3: {}", e)))?;
+
+    let mut stdout = String::new();
+    let mut stderr = String::new();
+
+    if let Some(mut out) = child.stdout.take() {
+        out.read_to_string(&mut stdout).await.ok();
+    }
+    if let Some(mut err) = child.stderr.take() {
+        err.read_to_string(&mut stderr).await.ok();
+    }
+
+    let logs = format!("{}{}", stdout, stderr);
+    let success = status.success();
+
+    let result = if success {
+        let trimmed = stdout.trim();
+        // Get the last line as result (in case there's debug output)
+        let last_line = trimmed.lines().last().unwrap_or("");
+        serde_json::from_str(last_line).unwrap_or_else(|_| serde_json::json!(trimmed))
+    } else {
+        serde_json::json!({ "error": stderr.trim() })
+    };
+
+    Ok(ExecutionResult {
+        success,
+        result,
+        logs,
+    })
+}
+
+/// Execute a Deno/TypeScript script
+async fn execute_deno(code: &str, args: &serde_json::Value) -> Result<ExecutionResult> {
+    // Wrap the code to handle args and return JSON result
+    let wrapped_code = format!(
+        r#"
+const args = {};
+
+{}
+
+// Call main if it exists
+if (typeof main === 'function') {{
+    const result = await main(args);
+    console.log(JSON.stringify(result));
+}}
+"#,
+        serde_json::to_string(args)?,
+        code
+    );
+
+    let mut child = Command::new("deno")
+        .arg("eval")
+        .arg("--unstable")
+        .arg(&wrapped_code)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| LocalError::Execution(format!("Failed to spawn deno: {}", e)))?;
+
+    let status = child
+        .wait()
+        .await
+        .map_err(|e| LocalError::Execution(format!("Failed to wait for deno: {}", e)))?;
+
+    let mut stdout = String::new();
+    let mut stderr = String::new();
+
+    if let Some(mut out) = child.stdout.take() {
+        out.read_to_string(&mut stdout).await.ok();
+    }
+    if let Some(mut err) = child.stderr.take() {
+        err.read_to_string(&mut stderr).await.ok();
+    }
+
+    let logs = format!("{}{}", stdout, stderr);
+    let success = status.success();
+
+    let result = if success {
+        let trimmed = stdout.trim();
+        let last_line = trimmed.lines().last().unwrap_or("");
+        serde_json::from_str(last_line).unwrap_or_else(|_| serde_json::json!(trimmed))
+    } else {
+        serde_json::json!({ "error": stderr.trim() })
+    };
+
+    Ok(ExecutionResult {
+        success,
+        result,
+        logs,
+    })
+}
+
+/// Execute a Bun/TypeScript script
+async fn execute_bun(code: &str, args: &serde_json::Value) -> Result<ExecutionResult> {
+    // Similar to Deno but using Bun
+    let wrapped_code = format!(
+        r#"
+const args = {};
+
+{}
+
+// Call main if it exists
+if (typeof main === 'function') {{
+    const result = await main(args);
+    console.log(JSON.stringify(result));
+}}
+"#,
+        serde_json::to_string(args)?,
+        code
+    );
+
+    let mut child = Command::new("bun")
+        .arg("eval")
+        .arg(&wrapped_code)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| LocalError::Execution(format!("Failed to spawn bun: {}", e)))?;
+
+    let status = child
+        .wait()
+        .await
+        .map_err(|e| LocalError::Execution(format!("Failed to wait for bun: {}", e)))?;
+
+    let mut stdout = String::new();
+    let mut stderr = String::new();
+
+    if let Some(mut out) = child.stdout.take() {
+        out.read_to_string(&mut stdout).await.ok();
+    }
+    if let Some(mut err) = child.stderr.take() {
+        err.read_to_string(&mut stderr).await.ok();
+    }
+
+    let logs = format!("{}{}", stdout, stderr);
+    let success = status.success();
+
+    let result = if success {
+        let trimmed = stdout.trim();
+        let last_line = trimmed.lines().last().unwrap_or("");
+        serde_json::from_str(last_line).unwrap_or_else(|_| serde_json::json!(trimmed))
+    } else {
+        serde_json::json!({ "error": stderr.trim() })
+    };
+
+    Ok(ExecutionResult {
+        success,
+        result,
+        logs,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_bash_execution() {
+        let result = execute_script(
+            ScriptLang::Bash,
+            "echo 42",
+            &serde_json::json!({}),
+        )
+        .await
+        .unwrap();
+
+        assert!(result.success);
+        // Output "42" is parsed as JSON number
+        assert_eq!(result.result, serde_json::json!(42));
+    }
+
+    #[tokio::test]
+    async fn test_bash_with_args() {
+        let result = execute_script(
+            ScriptLang::Bash,
+            "echo $NAME",
+            &serde_json::json!({"name": "world"}),
+        )
+        .await
+        .unwrap();
+
+        assert!(result.success);
+        assert_eq!(result.result, serde_json::json!("world"));
+    }
+
+    #[tokio::test]
+    async fn test_bash_json_output() {
+        let result = execute_script(
+            ScriptLang::Bash,
+            r#"echo '{"key": "value"}'"#,
+            &serde_json::json!({}),
+        )
+        .await
+        .unwrap();
+
+        assert!(result.success);
+        assert_eq!(result.result, serde_json::json!({"key": "value"}));
+    }
+}

backend/windmill-local/src/flow_executor.rs

@@ -0,0 +1,750 @@
+//! Flow executor for local mode
+//!
+//! This module implements flow execution using the real Windmill flow types
+//! from windmill-common, but with libSQL as the backend.
+
+use std::collections::HashMap;
+use chrono::Utc;
+use serde::{Deserialize, Serialize};
+use serde_json::value::RawValue;
+use uuid::Uuid;
+
+use windmill_common::flows::{
+    Branch, FlowModule, FlowModuleValue, FlowValue, InputTransform,
+};
+use windmill_common::scripts::ScriptLang as WmScriptLang;
+
+use crate::db::LocalDb;
+use crate::error::{LocalError, Result};
+use crate::executor::{execute_script, ExecutionResult};
+use crate::jobs::ScriptLang;
+
+/// Flow execution state
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FlowStatus {
+    pub step: usize,
+    pub modules: Vec<ModuleStatus>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub failure_module: Option<FailureModule>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub retry: Option<RetryStatus>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ModuleStatus {
+    pub id: String,
+    #[serde(rename = "type")]
+    pub status_type: ModuleStatusType,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub iterator: Option<IteratorStatus>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub branch_chosen: Option<BranchChosen>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub branchall: Option<BranchAllStatus>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub enum ModuleStatusType {
+    WaitingForPriorSteps,
+    WaitingForEvents,
+    WaitingForExecutor,
+    InProgress,
+    Success,
+    Failure,
+    Skipped,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IteratorStatus {
+    pub index: usize,
+    pub itered: Vec<serde_json::Value>,
+    pub args: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BranchChosen {
+    #[serde(rename = "type")]
+    pub branch_type: String,
+    pub branch: Option<usize>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BranchAllStatus {
+    pub branch: usize,
+    pub len: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FailureModule {
+    pub id: String,
+    pub error: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RetryStatus {
+    pub fail_count: u32,
+}
+
+/// Context passed through flow execution
+#[derive(Debug, Clone)]
+pub struct FlowContext {
+    pub flow_input: serde_json::Value,
+    pub previous_result: serde_json::Value,
+    pub results_by_id: HashMap<String, serde_json::Value>,
+}
+
+impl FlowContext {
+    pub fn new(flow_input: serde_json::Value) -> Self {
+        Self {
+            flow_input: flow_input.clone(),
+            previous_result: flow_input,
+            results_by_id: HashMap::new(),
+        }
+    }
+
+    /// Get the evaluation context for input transforms
+    pub fn to_eval_context(&self) -> serde_json::Value {
+        serde_json::json!({
+            "flow_input": self.flow_input,
+            "previous_result": self.previous_result,
+            "results": self.results_by_id,
+        })
+    }
+}
+
+/// Execute a flow and return the result
+pub async fn execute_flow(
+    db: &LocalDb,
+    flow_value: &FlowValue,
+    flow_input: serde_json::Value,
+) -> Result<(serde_json::Value, FlowStatus)> {
+    let mut ctx = FlowContext::new(flow_input);
+    let mut status = FlowStatus {
+        step: 0,
+        modules: Vec::new(),
+        failure_module: None,
+        retry: None,
+    };
+
+    // Execute modules sequentially
+    for (idx, module) in flow_value.modules.iter().enumerate() {
+        status.step = idx;
+
+        let module_status = ModuleStatus {
+            id: module.id.clone(),
+            status_type: ModuleStatusType::InProgress,
+            result: None,
+            iterator: None,
+            branch_chosen: None,
+            branchall: None,
+        };
+        status.modules.push(module_status);
+
+        tracing::info!("Executing flow module {}: {}", idx, module.id);
+
+        match execute_module(db, module, &mut ctx).await {
+            Ok(result) => {
+                // Update context with result
+                ctx.results_by_id.insert(module.id.clone(), result.clone());
+                ctx.previous_result = result.clone();
+
+                // Update status
+                if let Some(ms) = status.modules.last_mut() {
+                    ms.status_type = ModuleStatusType::Success;
+                    ms.result = Some(result);
+                }
+            }
+            Err(e) => {
+                // Module failed
+                tracing::error!("Flow module {} failed: {}", module.id, e);
+
+                if let Some(ms) = status.modules.last_mut() {
+                    ms.status_type = ModuleStatusType::Failure;
+                    ms.result = Some(serde_json::json!({"error": e.to_string()}));
+                }
+
+                status.failure_module = Some(FailureModule {
+                    id: module.id.clone(),
+                    error: e.to_string(),
+                });
+
+                // Check if continue_on_error is set
+                if !module.continue_on_error.unwrap_or(false) {
+                    return Ok((serde_json::json!({"error": e.to_string()}), status));
+                }
+            }
+        }
+    }
+
+    Ok((ctx.previous_result, status))
+}
+
+/// Execute a single flow module
+#[async_recursion::async_recursion]
+async fn execute_module(
+    db: &LocalDb,
+    module: &FlowModule,
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    // Check skip_if condition
+    if let Some(skip_if) = &module.skip_if {
+        let should_skip = evaluate_expr(&skip_if.expr, &ctx.to_eval_context())?;
+        if should_skip.as_bool().unwrap_or(false) {
+            tracing::info!("Skipping module {} due to skip_if condition", module.id);
+            return Ok(ctx.previous_result.clone());
+        }
+    }
+
+    // Parse the module value
+    let module_value: FlowModuleValue = serde_json::from_str(module.value.get())
+        .map_err(|e| LocalError::Execution(format!("Failed to parse module value: {}", e)))?;
+
+    match module_value {
+        FlowModuleValue::Identity => {
+            Ok(ctx.previous_result.clone())
+        }
+
+        FlowModuleValue::RawScript { content, language, input_transforms, .. } => {
+            let args = resolve_input_transforms(&input_transforms, ctx)?;
+            let lang = convert_script_lang(&language);
+            let result = execute_script(lang, &content, &args).await?;
+            if result.success {
+                Ok(result.result)
+            } else {
+                Err(LocalError::Execution(result.result.to_string()))
+            }
+        }
+
+        FlowModuleValue::Script { path, input_transforms, .. } => {
+            // In local mode, we don't have access to saved scripts
+            Err(LocalError::Execution(format!(
+                "Script references (path: {}) are not supported in local mode. Use rawscript instead.",
+                path
+            )))
+        }
+
+        FlowModuleValue::Flow { path, .. } => {
+            // In local mode, we don't have access to saved flows
+            Err(LocalError::Execution(format!(
+                "Flow references (path: {}) are not supported in local mode. Use inline modules instead.",
+                path
+            )))
+        }
+
+        FlowModuleValue::ForloopFlow { iterator, modules, skip_failures, parallel, .. } => {
+            execute_forloop(db, &iterator, &modules, skip_failures, parallel, ctx).await
+        }
+
+        FlowModuleValue::WhileloopFlow { modules, skip_failures, .. } => {
+            execute_whileloop(db, &modules, skip_failures, ctx).await
+        }
+
+        FlowModuleValue::BranchOne { branches, default, .. } => {
+            execute_branch_one(db, &branches, &default, ctx).await
+        }
+
+        FlowModuleValue::BranchAll { branches, parallel } => {
+            execute_branch_all(db, &branches, parallel, ctx).await
+        }
+
+        FlowModuleValue::FlowScript { .. } => {
+            Err(LocalError::Execution(
+                "FlowScript (internal reference) is not supported in local mode".to_string()
+            ))
+        }
+
+        FlowModuleValue::AIAgent { .. } => {
+            Err(LocalError::Execution(
+                "AIAgent is not supported in local mode".to_string()
+            ))
+        }
+    }
+}
+
+/// Execute a for loop
+#[async_recursion::async_recursion]
+async fn execute_forloop(
+    db: &LocalDb,
+    iterator: &InputTransform,
+    modules: &[FlowModule],
+    skip_failures: bool,
+    _parallel: bool,  // TODO: implement parallel execution
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    // Evaluate the iterator expression
+    let iter_value = evaluate_input_transform(iterator, ctx)?;
+
+    let items = match iter_value.as_array() {
+        Some(arr) => arr.clone(),
+        None => {
+            return Err(LocalError::Execution(
+                "For loop iterator must evaluate to an array".to_string()
+            ));
+        }
+    };
+
+    let mut results = Vec::new();
+
+    for (idx, item) in items.iter().enumerate() {
+        tracing::debug!("For loop iteration {} of {}", idx + 1, items.len());
+
+        // Create iteration context
+        let mut iter_ctx = FlowContext {
+            flow_input: ctx.flow_input.clone(),
+            previous_result: item.clone(),
+            results_by_id: ctx.results_by_id.clone(),
+        };
+
+        // Add iter context
+        iter_ctx.results_by_id.insert("iter".to_string(), serde_json::json!({
+            "index": idx,
+            "value": item,
+        }));
+
+        // Execute modules in sequence
+        let mut iter_result = item.clone();
+        let mut had_error = false;
+
+        for module in modules {
+            match execute_module(db, module, &mut iter_ctx).await {
+                Ok(result) => {
+                    iter_ctx.results_by_id.insert(module.id.clone(), result.clone());
+                    iter_ctx.previous_result = result.clone();
+                    iter_result = result;
+                }
+                Err(e) => {
+                    had_error = true;
+                    if skip_failures {
+                        tracing::warn!("For loop iteration {} failed (skipping): {}", idx, e);
+                        iter_result = serde_json::json!({"error": e.to_string()});
+                    } else {
+                        return Err(e);
+                    }
+                    break;
+                }
+            }
+        }
+
+        if !had_error || skip_failures {
+            results.push(iter_result);
+        }
+    }
+
+    Ok(serde_json::Value::Array(results))
+}
+
+/// Execute a while loop
+#[async_recursion::async_recursion]
+async fn execute_whileloop(
+    db: &LocalDb,
+    modules: &[FlowModule],
+    skip_failures: bool,
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    const MAX_ITERATIONS: usize = 1000;
+    let mut results = Vec::new();
+    let mut iteration = 0;
+
+    loop {
+        if iteration >= MAX_ITERATIONS {
+            return Err(LocalError::Execution(format!(
+                "While loop exceeded maximum iterations ({})", MAX_ITERATIONS
+            )));
+        }
+
+        // Execute modules
+        let mut iter_result = ctx.previous_result.clone();
+        let mut should_continue = true;
+
+        for module in modules {
+            match execute_module(db, module, ctx).await {
+                Ok(result) => {
+                    ctx.results_by_id.insert(module.id.clone(), result.clone());
+                    ctx.previous_result = result.clone();
+                    iter_result = result;
+                }
+                Err(e) => {
+                    if skip_failures {
+                        tracing::warn!("While loop iteration {} failed (skipping): {}", iteration, e);
+                        iter_result = serde_json::json!({"error": e.to_string()});
+                    } else {
+                        return Err(e);
+                    }
+                    should_continue = false;
+                    break;
+                }
+            }
+        }
+
+        results.push(iter_result);
+        iteration += 1;
+
+        // Check stop condition (result should be truthy to continue)
+        if !should_continue {
+            break;
+        }
+
+        // Check if the result indicates we should stop
+        let continue_loop = match &ctx.previous_result {
+            serde_json::Value::Bool(b) => *b,
+            serde_json::Value::Null => false,
+            serde_json::Value::Object(obj) => {
+                // Check for a "continue" field
+                obj.get("continue")
+                    .and_then(|v| v.as_bool())
+                    .unwrap_or(false)
+            }
+            _ => false,
+        };
+
+        if !continue_loop {
+            break;
+        }
+    }
+
+    Ok(serde_json::Value::Array(results))
+}
+
+/// Execute branch-one (if/else)
+#[async_recursion::async_recursion]
+async fn execute_branch_one(
+    db: &LocalDb,
+    branches: &[Branch],
+    default: &[FlowModule],
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    // Find the first matching branch
+    for (idx, branch) in branches.iter().enumerate() {
+        let condition = evaluate_expr(&branch.expr, &ctx.to_eval_context())?;
+        if condition.as_bool().unwrap_or(false) {
+            tracing::debug!("Branch {} matched", idx);
+            return execute_branch_modules(db, &branch.modules, ctx).await;
+        }
+    }
+
+    // No branch matched, execute default
+    tracing::debug!("No branch matched, executing default");
+    execute_branch_modules(db, default, ctx).await
+}
+
+/// Execute branch-all (parallel branches)
+#[async_recursion::async_recursion]
+async fn execute_branch_all(
+    db: &LocalDb,
+    branches: &[Branch],
+    _parallel: bool,  // TODO: implement true parallel execution
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    let mut results = Vec::new();
+
+    // Execute all branches (sequentially for now)
+    for (idx, branch) in branches.iter().enumerate() {
+        tracing::debug!("Executing branch {}", idx);
+
+        let mut branch_ctx = ctx.clone();
+        match execute_branch_modules(db, &branch.modules, &mut branch_ctx).await {
+            Ok(result) => {
+                results.push(result);
+            }
+            Err(e) => {
+                if branch.skip_failure {
+                    tracing::warn!("Branch {} failed (skipping): {}", idx, e);
+                    results.push(serde_json::json!({"error": e.to_string()}));
+                } else {
+                    return Err(e);
+                }
+            }
+        }
+    }
+
+    Ok(serde_json::Value::Array(results))
+}
+
+/// Execute a sequence of modules in a branch
+#[async_recursion::async_recursion]
+async fn execute_branch_modules(
+    db: &LocalDb,
+    modules: &[FlowModule],
+    ctx: &mut FlowContext,
+) -> Result<serde_json::Value> {
+    let mut result = ctx.previous_result.clone();
+
+    for module in modules {
+        result = execute_module(db, module, ctx).await?;
+        ctx.results_by_id.insert(module.id.clone(), result.clone());
+        ctx.previous_result = result.clone();
+    }
+
+    Ok(result)
+}
+
+/// Resolve input transforms to concrete arguments
+fn resolve_input_transforms(
+    transforms: &HashMap<String, InputTransform>,
+    ctx: &FlowContext,
+) -> Result<serde_json::Value> {
+    let mut args = serde_json::Map::new();
+
+    for (key, transform) in transforms {
+        let value = evaluate_input_transform(transform, ctx)?;
+        args.insert(key.clone(), value);
+    }
+
+    Ok(serde_json::Value::Object(args))
+}
+
+/// Evaluate an input transform
+fn evaluate_input_transform(
+    transform: &InputTransform,
+    ctx: &FlowContext,
+) -> Result<serde_json::Value> {
+    match transform {
+        InputTransform::Static { value } => {
+            serde_json::from_str(value.get())
+                .map_err(|e| LocalError::Execution(format!("Invalid static value: {}", e)))
+        }
+        InputTransform::Javascript { expr } => {
+            evaluate_expr(expr, &ctx.to_eval_context())
+        }
+        InputTransform::Ai => {
+            Err(LocalError::Execution("AI input transforms are not supported in local mode".to_string()))
+        }
+    }
+}
+
+/// Evaluate a JavaScript expression
+fn evaluate_expr(expr: &str, context: &serde_json::Value) -> Result<serde_json::Value> {
+    let expr = expr.trim();
+
+    // Handle comparison operators
+    if let Some(result) = try_evaluate_comparison(expr, context) {
+        return Ok(result);
+    }
+
+    // Handle simple variable references
+    if let Some(val) = resolve_path(expr, context) {
+        return Ok(val);
+    }
+
+    // Handle boolean literals
+    if expr == "true" {
+        return Ok(serde_json::Value::Bool(true));
+    }
+    if expr == "false" {
+        return Ok(serde_json::Value::Bool(false));
+    }
+
+    // Handle numeric literals
+    if let Ok(n) = expr.parse::<i64>() {
+        return Ok(serde_json::json!(n));
+    }
+    if let Ok(n) = expr.parse::<f64>() {
+        return Ok(serde_json::json!(n));
+    }
+
+    // Handle string literals
+    if (expr.starts_with('"') && expr.ends_with('"')) ||
+       (expr.starts_with('\'') && expr.ends_with('\'')) {
+        return Ok(serde_json::json!(&expr[1..expr.len()-1]));
+    }
+
+    // For complex expressions, we'd need a full JS runtime
+    tracing::warn!("Complex expression not evaluated: {}", expr);
+    Ok(serde_json::json!(expr))
+}
+
+/// Try to resolve a path like "flow_input.x" or "results.a.b"
+fn resolve_path(path: &str, context: &serde_json::Value) -> Option<serde_json::Value> {
+    let parts: Vec<&str> = path.split('.').collect();
+    if parts.is_empty() {
+        return None;
+    }
+
+    let mut current = context.get(parts[0])?;
+
+    for part in &parts[1..] {
+        current = current.get(*part)?;
+    }
+
+    Some(current.clone())
+}
+
+/// Try to evaluate a comparison expression
+fn try_evaluate_comparison(expr: &str, context: &serde_json::Value) -> Option<serde_json::Value> {
+    // Supported operators: >, <, >=, <=, ==, !=, ===, !==
+    let operators = ["===", "!==", ">=", "<=", "==", "!=", ">", "<"];
+
+    for op in operators {
+        if let Some(pos) = expr.find(op) {
+            let left = expr[..pos].trim();
+            let right = expr[pos + op.len()..].trim();
+
+            let left_val = evaluate_expr(left, context).ok()?;
+            let right_val = evaluate_expr(right, context).ok()?;
+
+            let result = match op {
+                ">" => compare_values(&left_val, &right_val, |a, b| a > b),
+                "<" => compare_values(&left_val, &right_val, |a, b| a < b),
+                ">=" => compare_values(&left_val, &right_val, |a, b| a >= b),
+                "<=" => compare_values(&left_val, &right_val, |a, b| a <= b),
+                "==" | "===" => Some(left_val == right_val),
+                "!=" | "!==" => Some(left_val != right_val),
+                _ => None,
+            };
+
+            return result.map(serde_json::Value::Bool);
+        }
+    }
+
+    // Try logical operators
+    if let Some(pos) = expr.find("&&") {
+        let left = expr[..pos].trim();
+        let right = expr[pos + 2..].trim();
+        let left_val = evaluate_expr(left, context).ok()?;
+        let right_val = evaluate_expr(right, context).ok()?;
+        return Some(serde_json::Value::Bool(
+            is_truthy(&left_val) && is_truthy(&right_val)
+        ));
+    }
+
+    if let Some(pos) = expr.find("||") {
+        let left = expr[..pos].trim();
+        let right = expr[pos + 2..].trim();
+        let left_val = evaluate_expr(left, context).ok()?;
+        let right_val = evaluate_expr(right, context).ok()?;
+        return Some(serde_json::Value::Bool(
+            is_truthy(&left_val) || is_truthy(&right_val)
+        ));
+    }
+
+    None
+}
+
+/// Compare two JSON values numerically
+fn compare_values<F>(left: &serde_json::Value, right: &serde_json::Value, cmp: F) -> Option<bool>
+where
+    F: Fn(f64, f64) -> bool,
+{
+    let left_num = value_to_number(left)?;
+    let right_num = value_to_number(right)?;
+    Some(cmp(left_num, right_num))
+}
+
+/// Convert a JSON value to a number
+fn value_to_number(val: &serde_json::Value) -> Option<f64> {
+    match val {
+        serde_json::Value::Number(n) => n.as_f64(),
+        serde_json::Value::String(s) => s.parse().ok(),
+        serde_json::Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),
+        _ => None,
+    }
+}
+
+/// Check if a value is truthy (JavaScript semantics)
+fn is_truthy(val: &serde_json::Value) -> bool {
+    match val {
+        serde_json::Value::Null => false,
+        serde_json::Value::Bool(b) => *b,
+        serde_json::Value::Number(n) => n.as_f64().map(|f| f != 0.0).unwrap_or(false),
+        serde_json::Value::String(s) => !s.is_empty(),
+        serde_json::Value::Array(a) => !a.is_empty(),
+        serde_json::Value::Object(_) => true,
+    }
+}
+
+/// Convert windmill-common ScriptLang to local ScriptLang
+fn convert_script_lang(lang: &WmScriptLang) -> ScriptLang {
+    match lang {
+        WmScriptLang::Deno => ScriptLang::Deno,
+        WmScriptLang::Python3 => ScriptLang::Python3,
+        WmScriptLang::Bash => ScriptLang::Bash,
+        WmScriptLang::Go => ScriptLang::Go,
+        WmScriptLang::Bun => ScriptLang::Bun,
+        WmScriptLang::Nativets => ScriptLang::Deno,  // Fallback to Deno
+        _ => ScriptLang::Bash,  // Default fallback
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_evaluate_simple_expr() {
+        let ctx = serde_json::json!({
+            "flow_input": {"x": 10},
+            "previous_result": 42,
+            "results": {"a": "hello"},
+        });
+
+        assert_eq!(
+            evaluate_expr("flow_input", &ctx).unwrap(),
+            serde_json::json!({"x": 10})
+        );
+        assert_eq!(
+            evaluate_expr("previous_result", &ctx).unwrap(),
+            serde_json::json!(42)
+        );
+        assert_eq!(
+            evaluate_expr("results.a", &ctx).unwrap(),
+            serde_json::json!("hello")
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.x", &ctx).unwrap(),
+            serde_json::json!(10)
+        );
+    }
+
+    #[test]
+    fn test_evaluate_comparison_expr() {
+        let ctx = serde_json::json!({
+            "flow_input": {"x": 10, "y": 5},
+            "previous_result": 42,
+        });
+
+        assert_eq!(
+            evaluate_expr("flow_input.x > 5", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.x < 5", &ctx).unwrap(),
+            serde_json::Value::Bool(false)
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.x >= 10", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.y == 5", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+        assert_eq!(
+            evaluate_expr("previous_result > 40", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+    }
+
+    #[test]
+    fn test_evaluate_logical_expr() {
+        let ctx = serde_json::json!({
+            "flow_input": {"a": true, "b": false},
+        });
+
+        // Simple boolean logic (complex expressions with mixed operators need parentheses support)
+        assert_eq!(
+            evaluate_expr("flow_input.a && flow_input.a", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.a && flow_input.b", &ctx).unwrap(),
+            serde_json::Value::Bool(false)
+        );
+        assert_eq!(
+            evaluate_expr("flow_input.b || flow_input.a", &ctx).unwrap(),
+            serde_json::Value::Bool(true)
+        );
+    }
+}

backend/windmill-local/src/jobs.rs

@@ -0,0 +1,495 @@
+//! Job types and operations for local mode
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::db::LocalDb;
+use crate::error::{LocalError, Result};
+
+/// Job kind (mirrors windmill-common JobKind)
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum JobKind {
+    Script,
+    Preview,
+    Flow,
+    FlowPreview,
+    Dependencies,
+    FlowDependencies,
+    ScriptHub,
+    Identity,
+    Http,
+    Graphql,
+    Postgresql,
+    Noop,
+    AppDependencies,
+    DeploymentCallback,
+    SingleScriptFlow,
+    FlowScript,
+    FlowNode,
+    AppScript,
+}
+
+impl JobKind {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            JobKind::Script => "script",
+            JobKind::Preview => "preview",
+            JobKind::Flow => "flow",
+            JobKind::FlowPreview => "flowpreview",
+            JobKind::Dependencies => "dependencies",
+            JobKind::FlowDependencies => "flowdependencies",
+            JobKind::ScriptHub => "script_hub",
+            JobKind::Identity => "identity",
+            JobKind::Http => "http",
+            JobKind::Graphql => "graphql",
+            JobKind::Postgresql => "postgresql",
+            JobKind::Noop => "noop",
+            JobKind::AppDependencies => "appdependencies",
+            JobKind::DeploymentCallback => "deploymentcallback",
+            JobKind::SingleScriptFlow => "singlescriptflow",
+            JobKind::FlowScript => "flowscript",
+            JobKind::FlowNode => "flownode",
+            JobKind::AppScript => "appscript",
+        }
+    }
+}
+
+/// Script language
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ScriptLang {
+    Python3,
+    Deno,
+    Go,
+    Bash,
+    Postgresql,
+    Nativets,
+    Bun,
+    Mysql,
+    Bigquery,
+    Snowflake,
+    Graphql,
+    Powershell,
+    Mssql,
+    Php,
+    Bunnative,
+    Rust,
+    Ansible,
+    Csharp,
+    Oracledb,
+    Nu,
+    Java,
+    Duckdb,
+}
+
+impl ScriptLang {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ScriptLang::Python3 => "python3",
+            ScriptLang::Deno => "deno",
+            ScriptLang::Go => "go",
+            ScriptLang::Bash => "bash",
+            ScriptLang::Postgresql => "postgresql",
+            ScriptLang::Nativets => "nativets",
+            ScriptLang::Bun => "bun",
+            ScriptLang::Mysql => "mysql",
+            ScriptLang::Bigquery => "bigquery",
+            ScriptLang::Snowflake => "snowflake",
+            ScriptLang::Graphql => "graphql",
+            ScriptLang::Powershell => "powershell",
+            ScriptLang::Mssql => "mssql",
+            ScriptLang::Php => "php",
+            ScriptLang::Bunnative => "bunnative",
+            ScriptLang::Rust => "rust",
+            ScriptLang::Ansible => "ansible",
+            ScriptLang::Csharp => "csharp",
+            ScriptLang::Oracledb => "oracledb",
+            ScriptLang::Nu => "nu",
+            ScriptLang::Java => "java",
+            ScriptLang::Duckdb => "duckdb",
+        }
+    }
+
+    pub fn from_str(s: &str) -> Option<Self> {
+        match s {
+            "python3" => Some(ScriptLang::Python3),
+            "deno" => Some(ScriptLang::Deno),
+            "go" => Some(ScriptLang::Go),
+            "bash" => Some(ScriptLang::Bash),
+            "postgresql" => Some(ScriptLang::Postgresql),
+            "nativets" => Some(ScriptLang::Nativets),
+            "bun" => Some(ScriptLang::Bun),
+            "mysql" => Some(ScriptLang::Mysql),
+            "bigquery" => Some(ScriptLang::Bigquery),
+            "snowflake" => Some(ScriptLang::Snowflake),
+            "graphql" => Some(ScriptLang::Graphql),
+            "powershell" => Some(ScriptLang::Powershell),
+            "mssql" => Some(ScriptLang::Mssql),
+            "php" => Some(ScriptLang::Php),
+            "bunnative" => Some(ScriptLang::Bunnative),
+            "rust" => Some(ScriptLang::Rust),
+            "ansible" => Some(ScriptLang::Ansible),
+            "csharp" => Some(ScriptLang::Csharp),
+            "oracledb" => Some(ScriptLang::Oracledb),
+            "nu" => Some(ScriptLang::Nu),
+            "java" => Some(ScriptLang::Java),
+            "duckdb" => Some(ScriptLang::Duckdb),
+            _ => None,
+        }
+    }
+}
+
+/// Job status for completed jobs
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum JobStatus {
+    Success,
+    Failure,
+    Canceled,
+    Skipped,
+}
+
+impl JobStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            JobStatus::Success => "success",
+            JobStatus::Failure => "failure",
+            JobStatus::Canceled => "canceled",
+            JobStatus::Skipped => "skipped",
+        }
+    }
+
+    pub fn from_str(s: &str) -> Option<Self> {
+        match s {
+            "success" => Some(JobStatus::Success),
+            "failure" => Some(JobStatus::Failure),
+            "canceled" => Some(JobStatus::Canceled),
+            "skipped" => Some(JobStatus::Skipped),
+            _ => None,
+        }
+    }
+}
+
+/// Preview job request (simplified from windmill-api Preview struct)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PreviewRequest {
+    pub content: String,
+    pub language: ScriptLang,
+    #[serde(default)]
+    pub args: serde_json::Value,
+    pub lock: Option<String>,
+    pub tag: Option<String>,
+}
+
+/// Flow preview request
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FlowPreviewRequest {
+    pub value: serde_json::Value,  // FlowValue as JSON
+    #[serde(default)]
+    pub args: serde_json::Value,
+    pub tag: Option<String>,
+}
+
+/// A queued job (combines v2_job and v2_job_queue)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct QueuedJob {
+    pub id: Uuid,
+    pub workspace_id: String,
+    pub kind: JobKind,
+    pub script_lang: Option<ScriptLang>,
+    pub raw_code: Option<String>,
+    pub raw_lock: Option<String>,
+    pub raw_flow: Option<serde_json::Value>,
+    pub args: serde_json::Value,
+    pub tag: String,
+    pub created_at: DateTime<Utc>,
+    pub scheduled_for: DateTime<Utc>,
+    pub running: bool,
+    pub parent_job: Option<Uuid>,
+    pub root_job: Option<Uuid>,
+    pub flow_step_id: Option<String>,
+}
+
+/// A completed job
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CompletedJob {
+    pub id: Uuid,
+    pub workspace_id: String,
+    pub status: JobStatus,
+    pub result: serde_json::Value,
+    pub started_at: Option<DateTime<Utc>>,
+    pub completed_at: DateTime<Utc>,
+    pub duration_ms: Option<i64>,
+}
+
+/// Push a preview job to the queue
+pub async fn push_preview(db: &LocalDb, req: PreviewRequest) -> Result<Uuid> {
+    let id = Uuid::new_v4();
+    let now = Utc::now().to_rfc3339();
+    let args_json = serde_json::to_string(&req.args)?;
+    let tag = req.tag.as_deref().unwrap_or("deno");
+
+    // Insert into v2_job
+    db.execute(
+        r#"
+        INSERT INTO v2_job (id, kind, script_lang, raw_code, raw_lock, args, tag, created_at)
+        VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
+        "#,
+        libsql::params![
+            id.to_string(),
+            JobKind::Preview.as_str(),
+            req.language.as_str(),
+            req.content,
+            req.lock,
+            args_json,
+            tag,
+            now.clone(),
+        ],
+    )
+    .await?;
+
+    // Insert into v2_job_queue
+    db.execute(
+        r#"
+        INSERT INTO v2_job_queue (id, tag, created_at, scheduled_for, running)
+        VALUES (?1, ?2, ?3, ?4, 0)
+        "#,
+        libsql::params![id.to_string(), tag, now.clone(), now],
+    )
+    .await?;
+
+    // Insert into v2_job_runtime (for heartbeat tracking)
+    db.execute(
+        "INSERT INTO v2_job_runtime (id) VALUES (?1)",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    // Insert into job_perms (simplified)
+    db.execute(
+        "INSERT INTO job_perms (job_id) VALUES (?1)",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    tracing::info!("Pushed preview job: {}", id);
+    Ok(id)
+}
+
+/// Push a flow preview job to the queue
+pub async fn push_flow_preview(db: &LocalDb, req: FlowPreviewRequest) -> Result<Uuid> {
+    let id = Uuid::new_v4();
+    let now = Utc::now().to_rfc3339();
+    let args_json = serde_json::to_string(&req.args)?;
+    let flow_json = serde_json::to_string(&req.value)?;
+    let tag = req.tag.as_deref().unwrap_or("flow");
+
+    // Insert into v2_job
+    db.execute(
+        r#"
+        INSERT INTO v2_job (id, kind, raw_flow, args, tag, created_at)
+        VALUES (?1, ?2, ?3, ?4, ?5, ?6)
+        "#,
+        libsql::params![
+            id.to_string(),
+            JobKind::FlowPreview.as_str(),
+            flow_json,
+            args_json,
+            tag,
+            now.clone(),
+        ],
+    )
+    .await?;
+
+    // Insert into v2_job_queue
+    db.execute(
+        r#"
+        INSERT INTO v2_job_queue (id, tag, created_at, scheduled_for, running)
+        VALUES (?1, ?2, ?3, ?4, 0)
+        "#,
+        libsql::params![id.to_string(), tag, now.clone(), now],
+    )
+    .await?;
+
+    // Insert into v2_job_runtime
+    db.execute(
+        "INSERT INTO v2_job_runtime (id) VALUES (?1)",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    // Insert into job_perms
+    db.execute(
+        "INSERT INTO job_perms (job_id) VALUES (?1)",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    // Insert initial flow status
+    db.execute(
+        "INSERT INTO v2_job_status (id, flow_status) VALUES (?1, '{}')",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    tracing::info!("Pushed flow preview job: {}", id);
+    Ok(id)
+}
+
+/// Get a completed job result (for polling)
+pub async fn get_completed_job(db: &LocalDb, id: Uuid) -> Result<Option<CompletedJob>> {
+    let mut rows = db
+        .query(
+            r#"
+            SELECT id, workspace_id, status, result, started_at, completed_at, duration_ms
+            FROM v2_job_completed
+            WHERE id = ?1
+            "#,
+            libsql::params![id.to_string()],
+        )
+        .await?;
+
+    if let Some(row) = rows.next().await? {
+        let status_str: String = row.get(2)?;
+        let status = JobStatus::from_str(&status_str)
+            .ok_or_else(|| LocalError::InvalidJobState(status_str))?;
+
+        let result_str: Option<String> = row.get(3)?;
+        let result: serde_json::Value = result_str
+            .map(|s| serde_json::from_str(&s))
+            .transpose()?
+            .unwrap_or(serde_json::Value::Null);
+
+        let started_at_str: Option<String> = row.get(4)?;
+        let started_at = started_at_str
+            .map(|s| DateTime::parse_from_rfc3339(&s).map(|dt| dt.with_timezone(&Utc)))
+            .transpose()
+            .ok()
+            .flatten();
+
+        let completed_at_str: String = row.get(5)?;
+        let completed_at = DateTime::parse_from_rfc3339(&completed_at_str)
+            .map(|dt| dt.with_timezone(&Utc))
+            .unwrap_or_else(|_| Utc::now());
+
+        let duration_ms: Option<i64> = row.get(6)?;
+
+        Ok(Some(CompletedJob {
+            id,
+            workspace_id: row.get(1)?,
+            status,
+            result,
+            started_at,
+            completed_at,
+            duration_ms,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Mark a job as completed with result
+pub async fn complete_job(
+    db: &LocalDb,
+    id: Uuid,
+    status: JobStatus,
+    result: serde_json::Value,
+    started_at: DateTime<Utc>,
+) -> Result<()> {
+    let now = Utc::now();
+    let duration_ms = (now - started_at).num_milliseconds();
+    let result_json = serde_json::to_string(&result)?;
+
+    db.execute(
+        r#"
+        INSERT INTO v2_job_completed (id, workspace_id, status, result, started_at, completed_at, duration_ms)
+        SELECT ?1, workspace_id, ?2, ?3, ?4, ?5, ?6
+        FROM v2_job WHERE id = ?1
+        "#,
+        libsql::params![
+            id.to_string(),
+            status.as_str(),
+            result_json,
+            started_at.to_rfc3339(),
+            now.to_rfc3339(),
+            duration_ms,
+        ],
+    )
+    .await?;
+
+    // Remove from queue
+    db.execute(
+        "DELETE FROM v2_job_queue WHERE id = ?1",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    tracing::info!("Completed job {}: {:?}", id, status);
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_push_preview() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        let req = PreviewRequest {
+            content: "export function main() { return 42; }".to_string(),
+            language: ScriptLang::Deno,
+            args: serde_json::json!({}),
+            lock: None,
+            tag: None,
+        };
+
+        let id = push_preview(&db, req).await.unwrap();
+
+        // Verify job exists in queue
+        let mut rows = db
+            .query(
+                "SELECT running FROM v2_job_queue WHERE id = ?1",
+                libsql::params![id.to_string()],
+            )
+            .await
+            .unwrap();
+
+        let row = rows.next().await.unwrap().unwrap();
+        let running: i64 = row.get(0).unwrap();
+        assert_eq!(running, 0);
+    }
+
+    #[tokio::test]
+    async fn test_complete_job() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        let req = PreviewRequest {
+            content: "export function main() { return 42; }".to_string(),
+            language: ScriptLang::Deno,
+            args: serde_json::json!({}),
+            lock: None,
+            tag: None,
+        };
+
+        let id = push_preview(&db, req).await.unwrap();
+        let started_at = Utc::now();
+
+        complete_job(
+            &db,
+            id,
+            JobStatus::Success,
+            serde_json::json!(42),
+            started_at,
+        )
+        .await
+        .unwrap();
+
+        // Verify job is in completed
+        let completed = get_completed_job(&db, id).await.unwrap().unwrap();
+        assert_eq!(completed.status, JobStatus::Success);
+        assert_eq!(completed.result, serde_json::json!(42));
+    }
+}

backend/windmill-local/src/lib.rs

@@ -0,0 +1,31 @@
+//! Windmill Local Mode
+//!
+//! This crate provides a minimal local mode for Windmill using libSQL (SQLite/Turso)
+//! instead of PostgreSQL. The goal is to support preview execution end-to-end
+//! with a lightweight, embedded database.
+//!
+//! ## Scope
+//! - Script preview execution
+//! - Flow preview execution
+//! - In-memory or file-based SQLite storage
+//! - Remote Turso database support for multi-writer scenarios
+//!
+//! ## Non-goals (for this experiment)
+//! - Full feature parity with PostgreSQL mode
+//! - Multi-worker support (single embedded worker)
+//! - Persistence of scripts/flows (only jobs)
+
+pub mod db;
+pub mod schema;
+pub mod jobs;
+pub mod queue;
+pub mod executor;
+pub mod flow_executor;
+pub mod worker;
+pub mod server;
+pub mod error;
+
+pub use db::LocalDb;
+pub use error::LocalError;
+pub use worker::Worker;
+pub use server::LocalServer;

backend/windmill-local/src/queue.rs

@@ -0,0 +1,296 @@
+//! Queue operations for local mode
+//!
+//! Since local mode uses a single worker, we don't need the complex
+//! `FOR UPDATE SKIP LOCKED` mechanism. Instead, we use simple atomic
+//! operations with the database lock.
+
+use chrono::{DateTime, Utc};
+use uuid::Uuid;
+
+use crate::db::LocalDb;
+use crate::error::{LocalError, Result};
+use crate::jobs::{JobKind, QueuedJob, ScriptLang};
+
+/// Pull the next job from the queue
+///
+/// This is simplified from the PostgreSQL version since we have a single worker
+/// and use the connection mutex for coordination.
+pub async fn pull_job(db: &LocalDb) -> Result<Option<QueuedJob>> {
+    let now = Utc::now().to_rfc3339();
+
+    // Get the next job (ordered by priority, then scheduled_for)
+    // We use a transaction-like approach: SELECT then UPDATE
+    let mut rows = db
+        .query(
+            r#"
+            SELECT q.id, j.workspace_id, j.kind, j.script_lang, j.raw_code, j.raw_lock,
+                   j.raw_flow, j.args, q.tag, j.created_at, q.scheduled_for,
+                   j.parent_job, j.root_job, j.flow_step_id
+            FROM v2_job_queue q
+            JOIN v2_job j ON q.id = j.id
+            WHERE q.running = 0 AND q.scheduled_for <= ?1
+            ORDER BY q.priority DESC, q.scheduled_for ASC
+            LIMIT 1
+            "#,
+            libsql::params![now],
+        )
+        .await?;
+
+    let Some(row) = rows.next().await? else {
+        return Ok(None);
+    };
+
+    let id_str: String = row.get(0)?;
+    let id = Uuid::parse_str(&id_str).map_err(|e| LocalError::InvalidJobState(e.to_string()))?;
+
+    // Mark as running
+    let started_at = Utc::now().to_rfc3339();
+    db.execute(
+        "UPDATE v2_job_queue SET running = 1, started_at = ?2 WHERE id = ?1",
+        libsql::params![id_str.clone(), started_at],
+    )
+    .await?;
+
+    // Parse the job fields
+    let kind_str: String = row.get(2)?;
+    let kind = match kind_str.as_str() {
+        "preview" => JobKind::Preview,
+        "flowpreview" => JobKind::FlowPreview,
+        "script" => JobKind::Script,
+        "flow" => JobKind::Flow,
+        "flowscript" => JobKind::FlowScript,
+        "flownode" => JobKind::FlowNode,
+        _ => JobKind::Preview, // Default
+    };
+
+    let lang_str: Option<String> = row.get(3)?;
+    let script_lang = lang_str.and_then(|s| ScriptLang::from_str(&s));
+
+    let raw_code: Option<String> = row.get(4)?;
+    let raw_lock: Option<String> = row.get(5)?;
+
+    let raw_flow_str: Option<String> = row.get(6)?;
+    let raw_flow = raw_flow_str
+        .map(|s| serde_json::from_str(&s))
+        .transpose()?;
+
+    let args_str: Option<String> = row.get(7)?;
+    let args: serde_json::Value = args_str
+        .map(|s| serde_json::from_str(&s))
+        .transpose()?
+        .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));
+
+    let tag: String = row.get(8)?;
+
+    let created_at_str: String = row.get(9)?;
+    let created_at = DateTime::parse_from_rfc3339(&created_at_str)
+        .map(|dt| dt.with_timezone(&Utc))
+        .unwrap_or_else(|_| Utc::now());
+
+    let scheduled_for_str: String = row.get(10)?;
+    let scheduled_for = DateTime::parse_from_rfc3339(&scheduled_for_str)
+        .map(|dt| dt.with_timezone(&Utc))
+        .unwrap_or_else(|_| Utc::now());
+
+    let parent_job_str: Option<String> = row.get(11)?;
+    let parent_job = parent_job_str.and_then(|s| Uuid::parse_str(&s).ok());
+
+    let root_job_str: Option<String> = row.get(12)?;
+    let root_job = root_job_str.and_then(|s| Uuid::parse_str(&s).ok());
+
+    let flow_step_id: Option<String> = row.get(13)?;
+
+    Ok(Some(QueuedJob {
+        id,
+        workspace_id: row.get(1)?,
+        kind,
+        script_lang,
+        raw_code,
+        raw_lock,
+        raw_flow,
+        args,
+        tag,
+        created_at,
+        scheduled_for,
+        running: true,
+        parent_job,
+        root_job,
+        flow_step_id,
+    }))
+}
+
+/// Get queue statistics
+pub async fn queue_stats(db: &LocalDb) -> Result<QueueStats> {
+    let mut rows = db
+        .query(
+            r#"
+            SELECT
+                COUNT(*) as total,
+                SUM(CASE WHEN running = 1 THEN 1 ELSE 0 END) as running,
+                SUM(CASE WHEN running = 0 THEN 1 ELSE 0 END) as pending
+            FROM v2_job_queue
+            "#,
+            (),
+        )
+        .await?;
+
+    let row = rows.next().await?.ok_or(LocalError::QueueEmpty)?;
+
+    Ok(QueueStats {
+        total: row.get::<i64>(0)? as u64,
+        running: row.get::<i64>(1).unwrap_or(0) as u64,
+        pending: row.get::<i64>(2).unwrap_or(0) as u64,
+    })
+}
+
+#[derive(Debug, Clone)]
+pub struct QueueStats {
+    pub total: u64,
+    pub running: u64,
+    pub pending: u64,
+}
+
+/// Update job heartbeat (ping)
+pub async fn ping_job(db: &LocalDb, id: Uuid) -> Result<()> {
+    let now = Utc::now().to_rfc3339();
+    db.execute(
+        "UPDATE v2_job_runtime SET ping = ?2 WHERE id = ?1",
+        libsql::params![id.to_string(), now],
+    )
+    .await?;
+    Ok(())
+}
+
+/// Update flow status for a running flow job
+pub async fn update_flow_status(
+    db: &LocalDb,
+    id: Uuid,
+    flow_status: &serde_json::Value,
+) -> Result<()> {
+    let status_json = serde_json::to_string(flow_status)?;
+    db.execute(
+        "UPDATE v2_job_status SET flow_status = ?2 WHERE id = ?1",
+        libsql::params![id.to_string(), status_json],
+    )
+    .await?;
+    Ok(())
+}
+
+/// Push a child job for flow execution
+pub async fn push_flow_child_job(
+    db: &LocalDb,
+    parent_id: Uuid,
+    root_id: Uuid,
+    step_id: &str,
+    kind: JobKind,
+    script_lang: Option<ScriptLang>,
+    raw_code: Option<&str>,
+    args: &serde_json::Value,
+) -> Result<Uuid> {
+    let id = Uuid::new_v4();
+    let now = Utc::now().to_rfc3339();
+    let args_json = serde_json::to_string(args)?;
+
+    // Insert into v2_job
+    db.execute(
+        r#"
+        INSERT INTO v2_job (id, kind, script_lang, raw_code, args, parent_job, root_job, flow_step_id, created_at)
+        VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
+        "#,
+        libsql::params![
+            id.to_string(),
+            kind.as_str(),
+            script_lang.map(|l| l.as_str()),
+            raw_code,
+            args_json,
+            parent_id.to_string(),
+            root_id.to_string(),
+            step_id,
+            now.clone(),
+        ],
+    )
+    .await?;
+
+    // Insert into v2_job_queue
+    db.execute(
+        r#"
+        INSERT INTO v2_job_queue (id, tag, created_at, scheduled_for, running)
+        VALUES (?1, 'flow', ?2, ?3, 0)
+        "#,
+        libsql::params![id.to_string(), now.clone(), now],
+    )
+    .await?;
+
+    // Insert into v2_job_runtime
+    db.execute(
+        "INSERT INTO v2_job_runtime (id) VALUES (?1)",
+        libsql::params![id.to_string()],
+    )
+    .await?;
+
+    Ok(id)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::jobs::{push_preview, PreviewRequest};
+
+    #[tokio::test]
+    async fn test_pull_job() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        // Push a job
+        let req = PreviewRequest {
+            content: "export function main() { return 42; }".to_string(),
+            language: ScriptLang::Deno,
+            args: serde_json::json!({}),
+            lock: None,
+            tag: None,
+        };
+        let pushed_id = push_preview(&db, req).await.unwrap();
+
+        // Pull it
+        let job = pull_job(&db).await.unwrap().unwrap();
+        assert_eq!(job.id, pushed_id);
+        assert!(job.running);
+        assert_eq!(job.kind, JobKind::Preview);
+
+        // Queue should now be empty (job is running)
+        let job2 = pull_job(&db).await.unwrap();
+        assert!(job2.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_queue_stats() {
+        let db = LocalDb::in_memory().await.unwrap();
+
+        // Initially empty
+        let stats = queue_stats(&db).await.unwrap();
+        assert_eq!(stats.total, 0);
+
+        // Push two jobs
+        let req = PreviewRequest {
+            content: "test".to_string(),
+            language: ScriptLang::Deno,
+            args: serde_json::json!({}),
+            lock: None,
+            tag: None,
+        };
+        push_preview(&db, req.clone()).await.unwrap();
+        push_preview(&db, req).await.unwrap();
+
+        let stats = queue_stats(&db).await.unwrap();
+        assert_eq!(stats.total, 2);
+        assert_eq!(stats.pending, 2);
+        assert_eq!(stats.running, 0);
+
+        // Pull one
+        pull_job(&db).await.unwrap();
+
+        let stats = queue_stats(&db).await.unwrap();
+        assert_eq!(stats.total, 2);
+        assert_eq!(stats.pending, 1);
+        assert_eq!(stats.running, 1);
+    }
+}

backend/windmill-local/src/schema.rs

@@ -0,0 +1,218 @@
+//! SQLite schema for local mode
+//!
+//! This is a minimal schema supporting preview job execution.
+//! Key differences from PostgreSQL:
+//! - ENUMs are TEXT with CHECK constraints
+//! - JSONB is JSON (stored as TEXT in SQLite)
+//! - Arrays are JSON arrays
+//! - No FOR UPDATE SKIP LOCKED (single worker, in-process coordination)
+
+/// SQL to create the minimal schema for local mode preview execution
+pub const SCHEMA: &str = r#"
+-- Job kinds (equivalent to PostgreSQL ENUM)
+-- Values: script, preview, flow, flowpreview, dependencies, flowdependencies,
+--         script_hub, identity, http, graphql, postgresql, noop, appdependencies,
+--         deploymentcallback, singlescriptflow, flowscript, flownode, appscript
+
+-- Job status (equivalent to PostgreSQL ENUM)
+-- Values: success, failure, canceled, skipped
+
+-- Script languages (equivalent to PostgreSQL ENUM)
+-- Values: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery,
+--         snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible,
+--         csharp, oracledb, nu, java, duckdb
+
+-- Main job table (minimal for preview)
+CREATE TABLE IF NOT EXISTS v2_job (
+    id TEXT PRIMARY KEY,  -- UUID as TEXT
+    workspace_id TEXT NOT NULL DEFAULT 'local',
+
+    -- Raw code for preview jobs
+    raw_code TEXT,
+    raw_lock TEXT,
+    raw_flow TEXT,  -- JSON for flow definitions
+
+    -- Job metadata
+    tag TEXT DEFAULT 'deno',
+    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    created_by TEXT NOT NULL DEFAULT 'local_user',
+
+    -- Permission context (simplified for local mode)
+    permissioned_as TEXT NOT NULL DEFAULT 'u/local_user',
+    permissioned_as_email TEXT DEFAULT 'local@windmill.local',
+
+    -- Job type info
+    kind TEXT NOT NULL DEFAULT 'preview' CHECK (kind IN (
+        'script', 'preview', 'flow', 'flowpreview', 'dependencies',
+        'flowdependencies', 'script_hub', 'identity', 'http', 'graphql',
+        'postgresql', 'noop', 'appdependencies', 'deploymentcallback',
+        'singlescriptflow', 'flowscript', 'flownode', 'appscript'
+    )),
+
+    -- Script execution details
+    script_lang TEXT CHECK (script_lang IN (
+        'python3', 'deno', 'go', 'bash', 'postgresql', 'nativets', 'bun',
+        'mysql', 'bigquery', 'snowflake', 'graphql', 'powershell', 'mssql',
+        'php', 'bunnative', 'rust', 'ansible', 'csharp', 'oracledb', 'nu',
+        'java', 'duckdb'
+    )),
+
+    -- Flow execution details
+    parent_job TEXT,  -- UUID reference
+    root_job TEXT,    -- UUID reference
+    flow_step INTEGER,
+    flow_step_id TEXT,
+    flow_innermost_root_job TEXT,
+
+    -- Execution settings
+    timeout INTEGER,
+    priority INTEGER DEFAULT 0,
+    same_worker INTEGER DEFAULT 0,  -- BOOLEAN as INTEGER
+    visible_to_owner INTEGER DEFAULT 1,
+
+    -- Arguments (JSON)
+    args TEXT,  -- JSON object
+
+    -- Pre-run error if validation failed
+    pre_run_error TEXT
+);
+
+-- Job queue table
+CREATE TABLE IF NOT EXISTS v2_job_queue (
+    id TEXT PRIMARY KEY,  -- UUID, references v2_job.id
+    workspace_id TEXT NOT NULL DEFAULT 'local',
+
+    -- Timestamps
+    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    started_at TEXT,
+    scheduled_for TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+
+    -- Queue state
+    running INTEGER NOT NULL DEFAULT 0,  -- BOOLEAN
+    canceled_by TEXT,
+    canceled_reason TEXT,
+
+    -- Suspend state (for approval flows)
+    suspend INTEGER DEFAULT 0,
+    suspend_until TEXT,
+
+    -- Execution settings
+    tag TEXT DEFAULT 'deno',
+    priority INTEGER DEFAULT 0,
+    worker TEXT,
+
+    FOREIGN KEY (id) REFERENCES v2_job(id)
+);
+
+-- Index for queue ordering (simulates queue_sort_v2)
+CREATE INDEX IF NOT EXISTS idx_queue_sort ON v2_job_queue (
+    priority DESC, scheduled_for ASC, tag
+) WHERE running = 0;
+
+-- Job runtime tracking (heartbeat/ping)
+CREATE TABLE IF NOT EXISTS v2_job_runtime (
+    id TEXT PRIMARY KEY,  -- UUID, references v2_job.id
+    ping TEXT,  -- Timestamp
+    memory_peak INTEGER,
+
+    FOREIGN KEY (id) REFERENCES v2_job(id)
+);
+
+-- Completed jobs with results
+CREATE TABLE IF NOT EXISTS v2_job_completed (
+    id TEXT PRIMARY KEY,  -- UUID, references v2_job.id
+    workspace_id TEXT NOT NULL DEFAULT 'local',
+
+    -- Timing
+    started_at TEXT,
+    completed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    duration_ms INTEGER,
+
+    -- Result
+    result TEXT,  -- JSON
+    result_columns TEXT,  -- JSON array of column names
+
+    -- Status
+    status TEXT NOT NULL DEFAULT 'success' CHECK (status IN (
+        'success', 'failure', 'canceled', 'skipped'
+    )),
+
+    -- Cancellation details
+    canceled_by TEXT,
+    canceled_reason TEXT,
+
+    -- Flow status (for flow jobs)
+    flow_status TEXT,  -- JSON
+
+    -- Execution details
+    memory_peak INTEGER,
+    worker TEXT,
+    deleted INTEGER DEFAULT 0,  -- BOOLEAN
+
+    FOREIGN KEY (id) REFERENCES v2_job(id)
+);
+
+-- Index for completed job lookup by workspace and time
+CREATE INDEX IF NOT EXISTS idx_completed_workspace_time ON v2_job_completed (
+    workspace_id, completed_at DESC
+);
+
+-- Flow status tracking (separate from completed to allow updates during execution)
+CREATE TABLE IF NOT EXISTS v2_job_status (
+    id TEXT PRIMARY KEY,  -- UUID, references v2_job.id
+    flow_status TEXT,  -- JSON object tracking flow module execution
+    flow_leaf_jobs TEXT,  -- JSON object
+    workflow_as_code_status TEXT,  -- JSON object
+
+    FOREIGN KEY (id) REFERENCES v2_job(id)
+);
+
+-- Simplified job permissions (for local mode, mostly unused)
+CREATE TABLE IF NOT EXISTS job_perms (
+    job_id TEXT PRIMARY KEY,
+    email TEXT DEFAULT 'local@windmill.local',
+    username TEXT DEFAULT 'local_user',
+    is_admin INTEGER DEFAULT 1,  -- BOOLEAN
+    is_operator INTEGER DEFAULT 0,  -- BOOLEAN
+    workspace_id TEXT DEFAULT 'local',
+    groups TEXT,  -- JSON array
+    folders TEXT,  -- JSON array of objects
+
+    FOREIGN KEY (job_id) REFERENCES v2_job(id)
+);
+
+-- Simple audit log (optional for local mode)
+CREATE TABLE IF NOT EXISTS audit (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    workspace_id TEXT DEFAULT 'local',
+    timestamp TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    username TEXT DEFAULT 'local_user',
+    operation TEXT NOT NULL,
+    action_kind TEXT CHECK (action_kind IN ('create', 'update', 'delete', 'execute')),
+    resource TEXT,
+    parameters TEXT  -- JSON
+);
+
+-- Job logs storage
+CREATE TABLE IF NOT EXISTS job_logs (
+    job_id TEXT PRIMARY KEY,
+    workspace_id TEXT DEFAULT 'local',
+    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    logs TEXT,
+    log_offset INTEGER DEFAULT 0,
+
+    FOREIGN KEY (job_id) REFERENCES v2_job(id)
+);
+"#;
+
+/// SQL to drop all tables (for testing/reset)
+pub const DROP_SCHEMA: &str = r#"
+DROP TABLE IF EXISTS job_logs;
+DROP TABLE IF EXISTS audit;
+DROP TABLE IF EXISTS job_perms;
+DROP TABLE IF EXISTS v2_job_status;
+DROP TABLE IF EXISTS v2_job_completed;
+DROP TABLE IF EXISTS v2_job_runtime;
+DROP TABLE IF EXISTS v2_job_queue;
+DROP TABLE IF EXISTS v2_job;
+"#;

backend/windmill-local/src/server.rs

@@ -0,0 +1,419 @@
+//! HTTP server for local mode
+//!
+//! Provides a minimal API compatible with Windmill's preview endpoints.
+
+use std::net::SocketAddr;
+use std::sync::Arc;
+use std::time::Duration;
+
+use axum::{
+    extract::{Path, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::{get, post},
+    Json, Router,
+};
+use serde::{Deserialize, Serialize};
+use tokio::sync::watch;
+use tower_http::cors::{Any, CorsLayer};
+use tower_http::trace::TraceLayer;
+use uuid::Uuid;
+
+use crate::db::LocalDb;
+use crate::error::Result;
+use crate::jobs::{
+    get_completed_job, push_flow_preview, push_preview, FlowPreviewRequest,
+    JobStatus, PreviewRequest, ScriptLang,
+};
+use crate::worker::Worker;
+
+/// Application state shared across handlers
+pub struct AppState {
+    pub db: Arc<LocalDb>,
+}
+
+/// Local server that runs the API and embedded worker
+pub struct LocalServer {
+    db: Arc<LocalDb>,
+    addr: SocketAddr,
+}
+
+impl LocalServer {
+    /// Create a new local server
+    pub async fn new(addr: SocketAddr) -> Result<Self> {
+        let db = Arc::new(LocalDb::in_memory().await?);
+        Ok(Self { db, addr })
+    }
+
+    /// Create a local server with a file-based database
+    pub async fn with_file(addr: SocketAddr, db_path: &str) -> Result<Self> {
+        let db = Arc::new(LocalDb::file(db_path).await?);
+        Ok(Self { db, addr })
+    }
+
+    /// Create a local server connected to a remote Turso database
+    pub async fn with_turso(addr: SocketAddr, url: &str, auth_token: &str) -> Result<Self> {
+        let db = Arc::new(LocalDb::turso_remote(url, auth_token).await?);
+        Ok(Self { db, addr })
+    }
+
+    /// Run the server
+    pub async fn run(self) -> Result<()> {
+        let (shutdown_tx, shutdown_rx) = watch::channel(false);
+
+        // Start the embedded worker
+        let worker_db = self.db.clone();
+        let worker_handle = tokio::spawn(async move {
+            let mut worker = Worker::new(worker_db, shutdown_rx);
+            if let Err(e) = worker.run().await {
+                tracing::error!("Worker error: {}", e);
+            }
+        });
+
+        // Build the router
+        let state = Arc::new(AppState { db: self.db });
+        let app = create_router(state);
+
+        // Run the server
+        tracing::info!("Local server listening on {}", self.addr);
+        let listener = tokio::net::TcpListener::bind(self.addr).await.unwrap();
+
+        // Handle graceful shutdown
+        let shutdown_signal = async move {
+            tokio::signal::ctrl_c()
+                .await
+                .expect("Failed to install CTRL+C signal handler");
+            tracing::info!("Shutdown signal received");
+            shutdown_tx.send(true).ok();
+        };
+
+        axum::serve(listener, app)
+            .with_graceful_shutdown(shutdown_signal)
+            .await
+            .unwrap();
+
+        // Wait for worker to finish
+        worker_handle.await.ok();
+
+        Ok(())
+    }
+}
+
+/// Create the API router
+fn create_router(state: Arc<AppState>) -> Router {
+    let cors = CorsLayer::new()
+        .allow_origin(Any)
+        .allow_methods(Any)
+        .allow_headers(Any);
+
+    Router::new()
+        // Health check
+        .route("/health", get(health_check))
+        // Preview endpoints (mimics windmill-api)
+        .route("/api/w/:workspace/jobs/run/preview", post(run_preview))
+        .route(
+            "/api/w/:workspace/jobs/run_wait_result/preview",
+            post(run_wait_result_preview),
+        )
+        .route(
+            "/api/w/:workspace/jobs/run/preview_flow",
+            post(run_preview_flow),
+        )
+        .route(
+            "/api/w/:workspace/jobs/run_wait_result/preview_flow",
+            post(run_wait_result_preview_flow),
+        )
+        // Get job result
+        .route(
+            "/api/w/:workspace/jobs_u/completed/get_result/:job_id",
+            get(get_job_result),
+        )
+        .layer(TraceLayer::new_for_http())
+        .layer(cors)
+        .with_state(state)
+}
+
+// === Request/Response Types ===
+
+#[derive(Debug, Deserialize)]
+struct PreviewPayload {
+    content: String,
+    language: String,
+    #[serde(default)]
+    args: serde_json::Value,
+    lock: Option<String>,
+    tag: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct FlowPreviewPayload {
+    value: serde_json::Value,
+    #[serde(default)]
+    args: serde_json::Value,
+    tag: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+struct JobCreatedResponse {
+    job_id: String,
+}
+
+#[derive(Debug, Serialize)]
+struct ErrorResponse {
+    error: String,
+}
+
+// === Handlers ===
+
+async fn health_check() -> &'static str {
+    "OK"
+}
+
+/// Run a preview job (async - returns job ID)
+async fn run_preview(
+    State(state): State<Arc<AppState>>,
+    Path(_workspace): Path<String>,
+    Json(payload): Json<PreviewPayload>,
+) -> impl IntoResponse {
+    let lang = match ScriptLang::from_str(&payload.language) {
+        Some(l) => l,
+        None => {
+            return (
+                StatusCode::BAD_REQUEST,
+                Json(ErrorResponse {
+                    error: format!("Unknown language: {}", payload.language),
+                }),
+            )
+                .into_response();
+        }
+    };
+
+    let req = PreviewRequest {
+        content: payload.content,
+        language: lang,
+        args: payload.args,
+        lock: payload.lock,
+        tag: payload.tag,
+    };
+
+    match push_preview(&state.db, req).await {
+        Ok(job_id) => (
+            StatusCode::CREATED,
+            Json(JobCreatedResponse {
+                job_id: job_id.to_string(),
+            }),
+        )
+            .into_response(),
+        Err(e) => (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(ErrorResponse {
+                error: e.to_string(),
+            }),
+        )
+            .into_response(),
+    }
+}
+
+/// Run a preview job and wait for result
+async fn run_wait_result_preview(
+    State(state): State<Arc<AppState>>,
+    Path(_workspace): Path<String>,
+    Json(payload): Json<PreviewPayload>,
+) -> impl IntoResponse {
+    let lang = match ScriptLang::from_str(&payload.language) {
+        Some(l) => l,
+        None => {
+            return (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({"error": format!("Unknown language: {}", payload.language)})),
+            )
+                .into_response();
+        }
+    };
+
+    let req = PreviewRequest {
+        content: payload.content,
+        language: lang,
+        args: payload.args,
+        lock: payload.lock,
+        tag: payload.tag,
+    };
+
+    let job_id = match push_preview(&state.db, req).await {
+        Ok(id) => id,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": e.to_string()})),
+            )
+                .into_response();
+        }
+    };
+
+    // Poll for result with timeout
+    wait_for_result(&state.db, job_id, Duration::from_secs(60)).await
+}
+
+/// Run a flow preview job (async - returns job ID)
+async fn run_preview_flow(
+    State(state): State<Arc<AppState>>,
+    Path(_workspace): Path<String>,
+    Json(payload): Json<FlowPreviewPayload>,
+) -> impl IntoResponse {
+    let req = FlowPreviewRequest {
+        value: payload.value,
+        args: payload.args,
+        tag: payload.tag,
+    };
+
+    match push_flow_preview(&state.db, req).await {
+        Ok(job_id) => (
+            StatusCode::CREATED,
+            Json(JobCreatedResponse {
+                job_id: job_id.to_string(),
+            }),
+        )
+            .into_response(),
+        Err(e) => (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(ErrorResponse {
+                error: e.to_string(),
+            }),
+        )
+            .into_response(),
+    }
+}
+
+/// Run a flow preview job and wait for result
+async fn run_wait_result_preview_flow(
+    State(state): State<Arc<AppState>>,
+    Path(_workspace): Path<String>,
+    Json(payload): Json<FlowPreviewPayload>,
+) -> impl IntoResponse {
+    let req = FlowPreviewRequest {
+        value: payload.value,
+        args: payload.args,
+        tag: payload.tag,
+    };
+
+    let job_id = match push_flow_preview(&state.db, req).await {
+        Ok(id) => id,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": e.to_string()})),
+            )
+                .into_response();
+        }
+    };
+
+    // Poll for result with timeout
+    wait_for_result(&state.db, job_id, Duration::from_secs(120)).await
+}
+
+/// Get the result of a completed job
+async fn get_job_result(
+    State(state): State<Arc<AppState>>,
+    Path((_workspace, job_id)): Path<(String, String)>,
+) -> impl IntoResponse {
+    let job_id = match Uuid::parse_str(&job_id) {
+        Ok(id) => id,
+        Err(_) => {
+            return (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({"error": "Invalid job ID"})),
+            )
+                .into_response();
+        }
+    };
+
+    match get_completed_job(&state.db, job_id).await {
+        Ok(Some(job)) => {
+            if job.status == JobStatus::Success {
+                (StatusCode::OK, Json(job.result)).into_response()
+            } else {
+                (StatusCode::INTERNAL_SERVER_ERROR, Json(job.result)).into_response()
+            }
+        }
+        Ok(None) => (
+            StatusCode::NOT_FOUND,
+            Json(serde_json::json!({"error": "Job not found or not completed"})),
+        )
+            .into_response(),
+        Err(e) => (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(serde_json::json!({"error": e.to_string()})),
+        )
+            .into_response(),
+    }
+}
+
+/// Poll for job completion with timeout
+async fn wait_for_result(
+    db: &LocalDb,
+    job_id: Uuid,
+    timeout: Duration,
+) -> axum::response::Response {
+    let start = std::time::Instant::now();
+    let fast_poll_duration = Duration::from_secs(2);
+    let fast_poll_interval = Duration::from_millis(50);
+    let slow_poll_interval = Duration::from_millis(200);
+
+    loop {
+        if start.elapsed() > timeout {
+            return (
+                StatusCode::REQUEST_TIMEOUT,
+                Json(serde_json::json!({"error": "Timeout waiting for job result"})),
+            )
+                .into_response();
+        }
+
+        match get_completed_job(db, job_id).await {
+            Ok(Some(job)) => {
+                if job.status == JobStatus::Success {
+                    return (StatusCode::OK, Json(job.result)).into_response();
+                } else {
+                    return (StatusCode::INTERNAL_SERVER_ERROR, Json(job.result)).into_response();
+                }
+            }
+            Ok(None) => {
+                // Job not completed yet, keep polling
+                let interval = if start.elapsed() < fast_poll_duration {
+                    fast_poll_interval
+                } else {
+                    slow_poll_interval
+                };
+                tokio::time::sleep(interval).await;
+            }
+            Err(e) => {
+                return (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(serde_json::json!({"error": e.to_string()})),
+                )
+                    .into_response();
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use axum::body::Body;
+    use axum::http::Request;
+    use tower::ServiceExt;
+
+    #[tokio::test]
+    async fn test_health_check() {
+        let db = Arc::new(LocalDb::in_memory().await.unwrap());
+        let state = Arc::new(AppState { db });
+        let app = create_router(state);
+
+        let response = app
+            .oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
+            .await
+            .unwrap();
+
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+}

backend/windmill-local/src/worker.rs

@@ -0,0 +1,204 @@
+//! Worker for local mode
+//!
+//! A single embedded worker that pulls jobs from the queue and executes them.
+
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::watch;
+use chrono::Utc;
+
+use crate::db::LocalDb;
+use crate::error::Result;
+use crate::executor::{execute_script, ExecutionResult};
+use crate::flow_executor;
+use crate::jobs::{complete_job, JobKind, JobStatus, QueuedJob};
+use crate::queue::pull_job;
+use windmill_common::flows::FlowValue;
+
+/// Worker that processes jobs from the queue
+pub struct Worker {
+    db: Arc<LocalDb>,
+    /// Channel to signal shutdown
+    shutdown_rx: watch::Receiver<bool>,
+}
+
+impl Worker {
+    /// Create a new worker
+    pub fn new(db: Arc<LocalDb>, shutdown_rx: watch::Receiver<bool>) -> Self {
+        Self { db, shutdown_rx }
+    }
+
+    /// Run the worker loop
+    pub async fn run(&mut self) -> Result<()> {
+        tracing::info!("Worker started");
+
+        loop {
+            // Check for shutdown signal
+            if *self.shutdown_rx.borrow() {
+                tracing::info!("Worker received shutdown signal");
+                break;
+            }
+
+            // Try to pull a job
+            match pull_job(&self.db).await {
+                Ok(Some(job)) => {
+                    tracing::info!("Processing job: {} (kind: {:?})", job.id, job.kind);
+                    if let Err(e) = self.process_job(job).await {
+                        tracing::error!("Error processing job: {}", e);
+                    }
+                }
+                Ok(None) => {
+                    // No jobs available, wait a bit before polling again
+                    tokio::select! {
+                        _ = tokio::time::sleep(Duration::from_millis(100)) => {}
+                        _ = self.shutdown_rx.changed() => {}
+                    }
+                }
+                Err(e) => {
+                    tracing::error!("Error pulling job: {}", e);
+                    tokio::time::sleep(Duration::from_millis(500)).await;
+                }
+            }
+        }
+
+        tracing::info!("Worker stopped");
+        Ok(())
+    }
+
+    /// Process a single job
+    async fn process_job(&self, job: QueuedJob) -> Result<()> {
+        let started_at = Utc::now();
+
+        match job.kind {
+            JobKind::Preview => {
+                self.process_preview_job(job, started_at).await
+            }
+            JobKind::FlowPreview => {
+                self.process_flow_preview_job(job, started_at).await
+            }
+            _ => {
+                // Unsupported job kind
+                let error_result = serde_json::json!({
+                    "error": format!("Unsupported job kind in local mode: {:?}", job.kind)
+                });
+                complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await
+            }
+        }
+    }
+
+    /// Process a script preview job
+    async fn process_preview_job(&self, job: QueuedJob, started_at: chrono::DateTime<Utc>) -> Result<()> {
+        let Some(code) = &job.raw_code else {
+            let error_result = serde_json::json!({"error": "No code provided for preview"});
+            return complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await;
+        };
+
+        let Some(lang) = job.script_lang else {
+            let error_result = serde_json::json!({"error": "No language specified for preview"});
+            return complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await;
+        };
+
+        // Execute the script
+        let exec_result = execute_script(lang, code, &job.args).await;
+
+        match exec_result {
+            Ok(ExecutionResult { success, result, logs }) => {
+                tracing::debug!("Job {} logs:\n{}", job.id, logs);
+                let status = if success { JobStatus::Success } else { JobStatus::Failure };
+                complete_job(&self.db, job.id, status, result, started_at).await
+            }
+            Err(e) => {
+                let error_result = serde_json::json!({"error": e.to_string()});
+                complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await
+            }
+        }
+    }
+
+    /// Process a flow preview job using the full flow executor
+    async fn process_flow_preview_job(&self, job: QueuedJob, started_at: chrono::DateTime<Utc>) -> Result<()> {
+        let Some(flow_json) = &job.raw_flow else {
+            let error_result = serde_json::json!({"error": "No flow definition provided"});
+            return complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await;
+        };
+
+        // Parse the flow value using windmill-common types
+        let flow_value: FlowValue = match serde_json::from_value(flow_json.clone()) {
+            Ok(fv) => fv,
+            Err(e) => {
+                let error_result = serde_json::json!({"error": format!("Failed to parse flow: {}", e)});
+                return complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await;
+            }
+        };
+
+        if flow_value.modules.is_empty() {
+            let error_result = serde_json::json!({"error": "Flow has no modules"});
+            return complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await;
+        }
+
+        tracing::info!("Executing flow with {} modules", flow_value.modules.len());
+
+        // Execute the flow using the full flow executor
+        match flow_executor::execute_flow(&self.db, &flow_value, job.args.clone()).await {
+            Ok((result, status)) => {
+                let is_failure = status.failure_module.is_some();
+                let final_result = serde_json::json!({
+                    "result": result,
+                    "flow_status": status
+                });
+
+                let job_status = if is_failure {
+                    JobStatus::Failure
+                } else {
+                    JobStatus::Success
+                };
+
+                complete_job(&self.db, job.id, job_status, final_result, started_at).await
+            }
+            Err(e) => {
+                let error_result = serde_json::json!({"error": e.to_string()});
+                complete_job(&self.db, job.id, JobStatus::Failure, error_result, started_at).await
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::jobs::{get_completed_job, push_preview, PreviewRequest, ScriptLang};
+
+    #[tokio::test]
+    async fn test_worker_processes_bash_preview() {
+        let db = Arc::new(LocalDb::in_memory().await.unwrap());
+        let (shutdown_tx, shutdown_rx) = watch::channel(false);
+
+        // Push a bash preview job
+        let req = PreviewRequest {
+            content: "echo 42".to_string(),
+            language: ScriptLang::Bash,
+            args: serde_json::json!({}),
+            lock: None,
+            tag: None,
+        };
+        let job_id = push_preview(&db, req).await.unwrap();
+
+        // Create and run worker for one iteration
+        let mut worker = Worker::new(db.clone(), shutdown_rx);
+
+        // Process one job then shutdown
+        tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_millis(500)).await;
+            shutdown_tx.send(true).unwrap();
+        });
+
+        worker.run().await.unwrap();
+
+        // Check the job completed
+        let completed = get_completed_job(&db, job_id).await.unwrap();
+        assert!(completed.is_some());
+        let completed = completed.unwrap();
+        assert_eq!(completed.status, JobStatus::Success);
+        // Output "42" is parsed as JSON number
+        assert_eq!(completed.result, serde_json::json!(42));
+    }
+}

backend/windmill-mcp/src/client/mod.rs

@@ -190,7 +190,7 @@ impl McpClient {
         args_str: &str,
     ) -> Result<Option<serde_json::Map<String, serde_json::Value>>> {
         if args_str.trim().is_empty() {
-            return Ok(None);
+            return Ok(Some(serde_json::Map::new()));
         }
 
         let args_value: serde_json::Value =
@@ -198,7 +198,7 @@ impl McpClient {
 
         match args_value {
             serde_json::Value::Object(map) => Ok(Some(map)),
-            serde_json::Value::Null => Ok(None),
+            serde_json::Value::Null => Ok(Some(serde_json::Map::new())),
             _ => Ok(Some(
                 vec![("value".to_string(), args_value)]
                     .into_iter()