nits

* feat: critical alert if disk near full

* update logic to cover edge-case

* update logic

* windows support

* Update README.md

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* make it periodically

* add extra safety

* respect killpills

* do not check every 3 secs

* move to monitor.rs

* rework

* alter readme

* extending functionality

* fix worker_mode

* fix compilation

* fix typo

* make use of AI suggestion

* update ee-repo-ref

* update ee ref

* logs are CE

* update ee repo ref

* remove systemstat from worker crate

* fix comp error + sync cargo.lock

* more comptime fixing

* Update ee-repo-ref.txt

* fix compilation error

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

.env

@@ -10,4 +10,4 @@ WM_IMAGE=ghcr.io/windmill-labs/windmill:main
 
 # To rotate logs, set the following variables:
 #LOG_MAX_SIZE=10m
-#LOG_MAX_FILE=3
+#LOG_MAX_FILE=3

.github/change-versions.sh

@@ -24,4 +24,4 @@ sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
 
 sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" ${root_dirpath}/backend/Cargo.lock
 
-cd ${root_dirpath}/frontend && npm i --package-lock-only
+cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts

.github/workflows/build-publish-rh-image.yml

@@ -64,7 +64,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp
           secrets: |
             rh_username=${{ secrets.RH_USERNAME }}
             rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
           platforms: linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp
           secrets: |
             rh_username=${{ secrets.RH_USERNAME }}
             rh_password=${{ secrets.RH_PASSWORD }}
@@ -111,8 +111,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: RHEL9-amd64 build
-          path:
-            ${{ steps.extract-ee-amd64.outputs.destination
+          path: ${{ steps.extract-ee-amd64.outputs.destination
             }}/windmill-ee-amd64-rhel9
 
       # - uses: actions/upload-artifact@v4

.github/workflows/build_windows_worker_.yml

@@ -51,7 +51,7 @@ jobs:
           $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
           mkdir frontend/build && cd backend
           New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
-          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
+          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,mcp
       - name: Rename binary with corresponding architecture
         run: |
           Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"

.github/workflows/docker-image-rpi4.yml

@@ -67,7 +67,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
+            features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
             ${{ steps.meta-public.outputs.tags }}

.github/workflows/docker-image.yml

@@ -1,10 +1,8 @@
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME:
-    ${{ github.event_name != 'pull_request' && github.event_name !=
+  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.event_name !=
     'workflow_dispatch' && github.repository || 'windmill-labs/windmill-test' }}
-  DEV_SHA:
-    ${{ github.event_name != 'pull_request' && github.event_name !=
+  DEV_SHA: ${{ github.event_name != 'pull_request' && github.event_name !=
     'workflow_dispatch' && 'dev' || github.event.inputs.tag || github.sha }}
 name: Build windmill:main
 on:
@@ -40,8 +38,7 @@ permissions: write-all
 jobs:
   build:
     runs-on: ubicloud
-    if:
-      (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
+    if: (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
       !github.event.inputs.ee)
     steps:
       - uses: actions/checkout@v4
@@ -95,7 +92,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages
+            features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
             ${{ steps.meta-public.outputs.tags }}
@@ -157,7 +154,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
             ${{ steps.meta-ee-public.outputs.tags }}
@@ -315,7 +312,7 @@ jobs:
     needs: [run_integration_test, build]
     if:
       github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' ||
-      startsWith(github.ref, 'refs/tags/v'))  && (github.event_name != 'workflow_dispatch') 
+      startsWith(github.ref, 'refs/tags/v'))  && (github.event_name != 'workflow_dispatch')
     steps:
       - uses: actions/checkout@v4
         with:
@@ -355,7 +352,7 @@ jobs:
   verify_ee_image_vulnerabilities:
     runs-on: ubicloud
     needs: [tag_latest_ee]
-    if: startsWith(github.ref, 'refs/tags/v') && (github.event_name != 'workflow_dispatch') 
+    if: startsWith(github.ref, 'refs/tags/v') && (github.event_name != 'workflow_dispatch')
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -397,8 +394,7 @@ jobs:
   build_ee_nsjail:
     needs: [build_ee]
     runs-on: ubicloud
-    if:
-      (github.event_name != 'pull_request') && ((github.event_name != 'workflow_dispatch') || (github.event.inputs.ee || github.event.inputs.nsjail))
+    if: (github.event_name != 'pull_request') && ((github.event_name != 'workflow_dispatch') || (github.event.inputs.ee || github.event.inputs.nsjail))
 
     steps:
       - uses: actions/checkout@v4
@@ -437,7 +433,7 @@ jobs:
         run: |
           sed -i 's|FROM ghcr.io/windmill-labs/windmill-ee:dev|FROM ghcr.io/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}|' ./docker/DockerfileNsjail
           cat ./docker/DockerfileNsjail | grep "FROM"
-        
+
       - name: Build and push publicly ee
         uses: depot/build-push-action@v1
         with:
@@ -451,12 +447,10 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-
   publish_ecr_s3:
     needs: [build_ee_nsjail]
     runs-on: ubicloud-standard-2-arm
-    if:
-      (github.event_name != 'pull_request') && (github.event_name !=
+    if: (github.event_name != 'pull_request') && (github.event_name !=
       'workflow_dispatch')
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/publish_windows_worker.yml

@@ -53,7 +53,7 @@ jobs:
           $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
           mkdir frontend/build && cd backend
           New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
-          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages
+          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,mcp
       - name: Rename binary with corresponding architecture
         run: |
           Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"

CHANGELOG.md

@@ -1,5 +1,172 @@
 # Changelog
 
+## [1.486.1](https://github.com/windmill-labs/windmill/compare/v1.486.0...v1.486.1) (2025-05-04)
+
+
+### Bug Fixes
+
+* improve MultiSelectWrapper behavior ([36da8ae](https://github.com/windmill-labs/windmill/commit/36da8aec080742e13f23e1dee12b3954947f53dd))
+
+## [1.486.0](https://github.com/windmill-labs/windmill/compare/v1.485.3...v1.486.0) (2025-05-01)
+
+
+### Features
+
+* add run now directly on schedule drawer and duplicate schedule option ([#5674](https://github.com/windmill-labs/windmill/issues/5674)) ([dfb947f](https://github.com/windmill-labs/windmill/commit/dfb947ff37c688f54a32de5aa3c5c3d142cb80f4))
+* Database Manager ([#5586](https://github.com/windmill-labs/windmill/issues/5586)) ([41c15fc](https://github.com/windmill-labs/windmill/commit/41c15fc78aaf844c559d3d6c772e04ecce436e9d))
+* Integrate MCP with hub ([#5685](https://github.com/windmill-labs/windmill/issues/5685)) ([ec701a9](https://github.com/windmill-labs/windmill/commit/ec701a9ee74c9d890b54234362392deca63a77c7))
+
+
+### Bug Fixes
+
+* Ai Chat: do not send tools if empty + respond even if tool fails ([#5692](https://github.com/windmill-labs/windmill/issues/5692)) ([9c55040](https://github.com/windmill-labs/windmill/commit/9c55040e47e76af8b7e2864b82fa30505545dcb5))
+* do not track relative deps for scripts with raw defined deps from CLI ([#5696](https://github.com/windmill-labs/windmill/issues/5696)) ([7eb9d7d](https://github.com/windmill-labs/windmill/commit/7eb9d7d46cb48ae69a3fd3ff852a57abae450a3b))
+* improve CLI file scanning performances ([0916978](https://github.com/windmill-labs/windmill/commit/09169784bd2d0ab7acf5f40dc86f36f1cae967b7))
+
+## [1.485.3](https://github.com/windmill-labs/windmill/compare/v1.485.2...v1.485.3) (2025-04-29)
+
+
+### Bug Fixes
+
+* improve performance of background cleanup monitoring  operations ([18dced3](https://github.com/windmill-labs/windmill/commit/18dced3c748cd5305f0934b26e50d69899563723))
+
+## [1.485.2](https://github.com/windmill-labs/windmill/compare/v1.485.1...v1.485.2) (2025-04-29)
+
+
+### Bug Fixes
+
+* improve agent workers for deployed scripts ([60018aa](https://github.com/windmill-labs/windmill/commit/60018aadf62cecadf111e019d3600513a89810f1))
+* make `#(extra_)requirements:` work better with pins ([#5680](https://github.com/windmill-labs/windmill/issues/5680)) ([1ab4160](https://github.com/windmill-labs/windmill/commit/1ab41603f4fd1526d0c944396ef250b184aed1f4))
+* **python:** handle better relative imports with requirements or extra_requirements ([f662cf5](https://github.com/windmill-labs/windmill/commit/f662cf5d75beed8fd114ba171cbe0fa8e4b2773f))
+
+## [1.485.1](https://github.com/windmill-labs/windmill/compare/v1.485.0...v1.485.1) (2025-04-28)
+
+
+### Bug Fixes
+
+* improve mcp mode api ([cf77ff0](https://github.com/windmill-labs/windmill/commit/cf77ff088b8382b861113120589de58f7cf241d0))
+* MCP handle long names + invalid char in prop key + fix for not found resource type ([#5668](https://github.com/windmill-labs/windmill/issues/5668)) ([eadae95](https://github.com/windmill-labs/windmill/commit/eadae95a42d679bf8792bdefd8b9d19dbcbc4b57))
+* skip_flow_update for dependency tracking table ([#5670](https://github.com/windmill-labs/windmill/issues/5670)) ([35b69da](https://github.com/windmill-labs/windmill/commit/35b69da25c5bd17deff5a54b635e9150cb865cc0))
+
+## [1.485.0](https://github.com/windmill-labs/windmill/compare/v1.484.0...v1.485.0) (2025-04-28)
+
+
+### Features
+
+* add universal search to object viewer ([7254743](https://github.com/windmill-labs/windmill/commit/72547437fead0a071fceac27dae8628cdcae6a3e))
+
+
+### Bug Fixes
+
+* add svelte 5 boundaries to app components to contain errors ([1b16918](https://github.com/windmill-labs/windmill/commit/1b1691837a7e6b88afbacf7d88c14ca5e475b493))
+* Fix object handling on some MCP clients + better frontend for MCP ([#5663](https://github.com/windmill-labs/windmill/issues/5663)) ([12c3202](https://github.com/windmill-labs/windmill/commit/12c32026e5879a65fc0f1cc9f2481087c4b95111))
+
+## [1.484.0](https://github.com/windmill-labs/windmill/compare/v1.483.2...v1.484.0) (2025-04-26)
+
+
+### Features
+
+* Add MCP endpoints ([#5639](https://github.com/windmill-labs/windmill/issues/5639)) ([a34ac4f](https://github.com/windmill-labs/windmill/commit/a34ac4fa24c2a5482e45724e76316d57f64f7040))
+* Add MCP only mode ([#5661](https://github.com/windmill-labs/windmill/issues/5661)) ([1625524](https://github.com/windmill-labs/windmill/commit/162552431138d68002c7060cad4ae31f1ec4c69c))
+* Ansible improvements (vault, roles and git repos) ([#5655](https://github.com/windmill-labs/windmill/issues/5655)) ([fdd1642](https://github.com/windmill-labs/windmill/commit/fdd1642ce10866da1d8d373bda44f050e2e0f403))
+
+
+### Bug Fixes
+
+* check for valid teams_channel config when saving critical alerts settings ([#5660](https://github.com/windmill-labs/windmill/issues/5660)) ([dc5c8d8](https://github.com/windmill-labs/windmill/commit/dc5c8d8c5f8577b7ded3da1d684cdb735fa7a936))
+* Fix CI for MCP + optimization ([#5657](https://github.com/windmill-labs/windmill/issues/5657)) ([b199a77](https://github.com/windmill-labs/windmill/commit/b199a77d486c5bfd086ca73a58a14bb747e386b5))
+* fix token creation after mcp mode change to make it non workspace specific ([2b5dfcf](https://github.com/windmill-labs/windmill/commit/2b5dfcfb251471dcc39b04c54e25008d617cc34f))
+* improve full-scaleout of autoscaling event logging ([8435eb3](https://github.com/windmill-labs/windmill/commit/8435eb3adff8429a73db88b12204f7cf8f14d3d2))
+* improve skip failure on parallel branchall ([a7b2b51](https://github.com/windmill-labs/windmill/commit/a7b2b51444d757964560de3a89024b1c9b0fefe9))
+
+## [1.483.2](https://github.com/windmill-labs/windmill/compare/v1.483.1...v1.483.2) (2025-04-23)
+
+
+### Bug Fixes
+
+* batch reruns query missing workspace_id check in subquery ([#5652](https://github.com/windmill-labs/windmill/issues/5652)) ([444a6ab](https://github.com/windmill-labs/windmill/commit/444a6abad670114c52e44f3606bf6fefc5d3fd98))
+* **frontend:** fix validity check ([#5654](https://github.com/windmill-labs/windmill/issues/5654)) ([c41c1eb](https://github.com/windmill-labs/windmill/commit/c41c1eb587bf22364f1202310a0c64b6040ab968))
+* improve MySQL datetime parser timezone handling (WIN-1155) ([#5645](https://github.com/windmill-labs/windmill/issues/5645)) ([5bca8f6](https://github.com/windmill-labs/windmill/commit/5bca8f60e970cc67839edb5dc491685f36cf0499))
+* track relative imports in python and ts even if lockfile is provided ([e316dbd](https://github.com/windmill-labs/windmill/commit/e316dbd9bdd5c59e9aaba6a4472bb7d832834e84))
+
+## [1.483.1](https://github.com/windmill-labs/windmill/compare/v1.483.0...v1.483.1) (2025-04-19)
+
+
+### Bug Fixes
+
+* pin libxml to 0.3.3 ([e5595e4](https://github.com/windmill-labs/windmill/commit/e5595e41b5c87704d814eff95bc00b82195728ba))
+
+## [1.483.0](https://github.com/windmill-labs/windmill/compare/v1.482.1...v1.483.0) (2025-04-19)
+
+
+### Features
+
+* handle different aws auth resource type ([#5637](https://github.com/windmill-labs/windmill/issues/5637)) ([5b123b0](https://github.com/windmill-labs/windmill/commit/5b123b01a1318208450789b5bcade447a0b331c7))
+* oidc support for sqs trigger ([#5614](https://github.com/windmill-labs/windmill/issues/5614)) ([34b307b](https://github.com/windmill-labs/windmill/commit/34b307b2be1f6cf92a81694325f4c333bdd7b055))
+
+
+### Bug Fixes
+
+* fix click outside popover fullscreen ([#5631](https://github.com/windmill-labs/windmill/issues/5631)) ([0811457](https://github.com/windmill-labs/windmill/commit/081145726a5d4ab81510a7637126a036823a1565))
+* improve flow editor step switch performance ([58fa4c8](https://github.com/windmill-labs/windmill/commit/58fa4c80062a5704bbd13ddda1b2f00c7c9e40dd))
+* linter in early stop doesn't include flow_input ([#5638](https://github.com/windmill-labs/windmill/issues/5638)) ([6a9bdfd](https://github.com/windmill-labs/windmill/commit/6a9bdfd3bd52ff802b6a71c3ae9504bfe7d0421f))
+* output picker output opening doesn't change id ([#5641](https://github.com/windmill-labs/windmill/issues/5641)) ([64c72b6](https://github.com/windmill-labs/windmill/commit/64c72b6fce669e47f04dc620750840857cbe66cf))
+
+## [1.482.1](https://github.com/windmill-labs/windmill/compare/v1.482.0...v1.482.1) (2025-04-16)
+
+
+### Bug Fixes
+
+* flow editor workspace script test use actual workspace script hash ([24e893b](https://github.com/windmill-labs/windmill/commit/24e893b8c50fafdb41f4b6e1777cb34aceafc466))
+* **frontend:** postgres remove selectedTable ([#5386](https://github.com/windmill-labs/windmill/issues/5386)) ([bd7c6a2](https://github.com/windmill-labs/windmill/commit/bd7c6a2a46047de5fe89753decdfdf1f4851ee3f))
+* **openapi:** fix openapi def of batch re-run jobs ([cb8731e](https://github.com/windmill-labs/windmill/commit/cb8731e7e37fb6cd052f5dae6fdce46e6ca2409c))
+* show workspace color if superadmin and not in workspace + change workspace name when switching workspace ([#5625](https://github.com/windmill-labs/windmill/issues/5625)) ([cc4384f](https://github.com/windmill-labs/windmill/commit/cc4384f48cc89f883237a2082d854d69a7b5dc56))
+
+## [1.482.0](https://github.com/windmill-labs/windmill/compare/v1.481.0...v1.482.0) (2025-04-15)
+
+
+### Features
+
+* add diff toggle to flow inline scripts ([#5550](https://github.com/windmill-labs/windmill/issues/5550)) ([b3ecde3](https://github.com/windmill-labs/windmill/commit/b3ecde3316252bcd7323de98149786349019ba7e))
+* add gcp trigger ([#5501](https://github.com/windmill-labs/windmill/issues/5501)) ([6339775](https://github.com/windmill-labs/windmill/commit/63397754046eed41d32e28d4698db37b4c9b9710))
+* add wildcards filter for worker/label/tags ([62f14d1](https://github.com/windmill-labs/windmill/commit/62f14d1cb95e3f1c7de85c46e1c6bb092247656c))
+* add windmill context to autocomplete ([#5548](https://github.com/windmill-labs/windmill/issues/5548)) ([b47c151](https://github.com/windmill-labs/windmill/commit/b47c15165f93ca68a58f81cf2b86fc9467155482))
+* agent workers v2 using http ([#5588](https://github.com/windmill-labs/windmill/issues/5588)) ([63fa499](https://github.com/windmill-labs/windmill/commit/63fa4990153f33434b49269922f7803d04e407cd))
+* Batch re-run ([#5553](https://github.com/windmill-labs/windmill/issues/5553)) ([26b5ea5](https://github.com/windmill-labs/windmill/commit/26b5ea5023a100c57d077910c99a5e5703edf1c1))
+* **frontend:** app editor code input component (monaco) ([#5566](https://github.com/windmill-labs/windmill/issues/5566)) ([177e16b](https://github.com/windmill-labs/windmill/commit/177e16bb18eed0d1c454b967aaa59547f61e8d26))
+* handle sending selected lines to ai context ([#5527](https://github.com/windmill-labs/windmill/issues/5527)) ([5abdc3e](https://github.com/windmill-labs/windmill/commit/5abdc3e4403b5c604309bd99a24d7a2847a17b9b))
+* Implement sending diff to ai ([#5510](https://github.com/windmill-labs/windmill/issues/5510)) ([e118d2c](https://github.com/windmill-labs/windmill/commit/e118d2cd5f9c641884a76229802a5228ef41f1a5))
+* make azure a standalone AI provider ([#5558](https://github.com/windmill-labs/windmill/issues/5558)) ([2c5e58c](https://github.com/windmill-labs/windmill/commit/2c5e58cf1ab9225d516540b38d9e4dde482a3a7f))
+* migrate to svelte5 + vite6 ([#4813](https://github.com/windmill-labs/windmill/issues/4813)) ([3c99b3f](https://github.com/windmill-labs/windmill/commit/3c99b3fdc7b78b1cdc7d8fb21d999296695f7889))
+* **postgres-trigger:** postgres trigger fix circular dependencies and add remove associate resource ([#5606](https://github.com/windmill-labs/windmill/issues/5606)) ([1daeb2f](https://github.com/windmill-labs/windmill/commit/1daeb2f48f3026621b3ffc58e10f048d5911906c))
+* **python:** per import requirement pin ([#5520](https://github.com/windmill-labs/windmill/issues/5520)) ([0b6d017](https://github.com/windmill-labs/windmill/commit/0b6d017fedc31e790a76cf29a1adaaf2a72acc61))
+* signed s3 objects ([#5593](https://github.com/windmill-labs/windmill/issues/5593)) ([b9e8796](https://github.com/windmill-labs/windmill/commit/b9e879618bc223ce17effde8bb4c5d1df2ad6df5))
+
+
+### Bug Fixes
+
+* add support for ${} syntax without default in bash ([#5594](https://github.com/windmill-labs/windmill/issues/5594)) ([3950cfd](https://github.com/windmill-labs/windmill/commit/3950cfd7e3297d7f8ec56430d6462f6b67ecd3c2))
+* app editor svelte 5 fixes ([#5570](https://github.com/windmill-labs/windmill/issues/5570)) ([b926076](https://github.com/windmill-labs/windmill/commit/b9260769883348ecd5aeb5684f527a8bf0073928))
+* binding not working in nested array script arg ([#5585](https://github.com/windmill-labs/windmill/issues/5585)) ([f5d46d5](https://github.com/windmill-labs/windmill/commit/f5d46d5751bc875b7f4da1db06be40571ac55ab8))
+* **cli:** properly handle enabled/disabled updates of schedules ([2629458](https://github.com/windmill-labs/windmill/commit/26294584d6c2ca02bbc4fc5f28cb8df6a5fb3790))
+* **cli:** wmill-locks improvement ([8d062c4](https://github.com/windmill-labs/windmill/commit/8d062c47ecd9e84a81140d5c59814da9217dd434))
+* Dynamic select does not work with tag //native ([#5576](https://github.com/windmill-labs/windmill/issues/5576)) ([1f3e7d9](https://github.com/windmill-labs/windmill/commit/1f3e7d9029051832db6ab1755b3cad38176a9e96)), closes [#5490](https://github.com/windmill-labs/windmill/issues/5490)
+* fix list jobs by tag ([0c3cb37](https://github.com/windmill-labs/windmill/commit/0c3cb3700a3fb9b69e396487bd7491dbbd8861c0))
+* flow editor svelte 5 issues ([#5567](https://github.com/windmill-labs/windmill/issues/5567)) ([4f6be6e](https://github.com/windmill-labs/windmill/commit/4f6be6ed340e26bf1ed95398a9dc9f1eb41b33dd))
+* freeze when clicking script history diff button ([#5581](https://github.com/windmill-labs/windmill/issues/5581)) ([07094b6](https://github.com/windmill-labs/windmill/commit/07094b6aa21f10688b138d2a81d4fd5833f003fc))
+* **frontend:** app builder - force json configuration in rich result ([#5565](https://github.com/windmill-labs/windmill/issues/5565)) ([6fae3a5](https://github.com/windmill-labs/windmill/commit/6fae3a566be06dae88ece8ec23f5723cd8f3f2b9))
+* **frontend:** load all step jobs ([#5617](https://github.com/windmill-labs/windmill/issues/5617)) ([16bed59](https://github.com/windmill-labs/windmill/commit/16bed593dfd0b735a92d0928df5091547b98ae79))
+* **frontend:** prevent deploy popover to show if deploy dropdown is open ([#5542](https://github.com/windmill-labs/windmill/issues/5542)) ([c2180c6](https://github.com/windmill-labs/windmill/commit/c2180c6eb34e14fe2292ff40aa6a99c627698d5e))
+* **frontend:** proper each block binding + better app settings reactivity ([#5568](https://github.com/windmill-labs/windmill/issues/5568)) ([4c71af8](https://github.com/windmill-labs/windmill/commit/4c71af8a74627d0ba76917e0dac0ac9e5e984cca))
+* improve app image picker UX ([#5589](https://github.com/windmill-labs/windmill/issues/5589)) ([f497a4b](https://github.com/windmill-labs/windmill/commit/f497a4bfae8d1bff097e0c2c9df8381a531dfeb9))
+* legacy script gen model selection ([#5574](https://github.com/windmill-labs/windmill/issues/5574)) ([3507925](https://github.com/windmill-labs/windmill/commit/3507925624a43804a3be463b6f7913cea5821384))
+* mssql ca_cert deserializing ([#5587](https://github.com/windmill-labs/windmill/issues/5587)) ([b4f8c88](https://github.com/windmill-labs/windmill/commit/b4f8c88c19bd4f844c3ecb53ececc340ee326b0e))
+* number input in app multiselect yields NOT_NUMBER ([#5616](https://github.com/windmill-labs/windmill/issues/5616)) ([4aae6ab](https://github.com/windmill-labs/windmill/commit/4aae6ab634280adc1de9abd890100b7c12c89158))
+* prevent invalid returned ai completion object errors ([#5564](https://github.com/windmill-labs/windmill/issues/5564)) ([9276c71](https://github.com/windmill-labs/windmill/commit/9276c717a21aaee3241845a9cc00d3fb6bce9eb9))
+* Remaining svelte 5 bugs ([#5563](https://github.com/windmill-labs/windmill/issues/5563)) ([6e9ec63](https://github.com/windmill-labs/windmill/commit/6e9ec6323c265a747ef8696865297e6d47abb016))
+* tenant id to never be undefined on teams ([#5572](https://github.com/windmill-labs/windmill/issues/5572)) ([102b58a](https://github.com/windmill-labs/windmill/commit/102b58a5f40dde22f15700d4b6c11eb7f3fbf4bb))
+* validate saved module before passing to flow module editor ([#5580](https://github.com/windmill-labs/windmill/issues/5580)) ([2eb1a16](https://github.com/windmill-labs/windmill/commit/2eb1a161d15627b440195b65eec54998561f4ef6))
+
 ## [1.481.0](https://github.com/windmill-labs/windmill/compare/v1.480.1...v1.481.0) (2025-04-02)
 
 

Caddyfile

@@ -12,7 +12,7 @@
         bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
         # reverse_proxy /ws_mp/* http://multiplayer:3002
-        # reverse_proxy /api/srch/* http://windmill_indexer:8001
+        # reverse_proxy /api/srch/* http://windmill_indexer:8002
         reverse_proxy /* http://windmill_server:8000
         # tls /certs/cert.pem /certs/key.pem
 }

Dockerfile

@@ -25,6 +25,7 @@ FROM node:20-alpine as frontend
 # install dependencies
 WORKDIR /frontend
 COPY ./frontend/package.json ./frontend/package-lock.json ./
+COPY ./frontend/scripts/ ./scripts/
 RUN npm ci
 
 # Copy all local files into the image.

README.md

@@ -363,6 +363,7 @@ you to have it being synced automatically everyday.
 | DENO_AUTH_TOKENS                    | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
 | DISABLE_RESPONSE_LOGS               | false                  | Disable response logs                                                                                                                                                                              | Server                |
 | CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true                   | If true, only superadmins can create new workspaces                                                                                                                                                | Server                |
+| MIN_FREE_DISK_SPACE_MB              | 15000                  | Minimum amount of free space on worker. Sends critical alert if worker has less free space.                                                                                                        | Worker                |
 
 ## Run a local dev setup
 

backend/.sqlx/query-00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        DELETE\n        FROM parallel_monitor_lock\n        WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval \n        RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q\n            WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL\n        ) AS workspace_id\n        ",
+  "query": "\n        DELETE\n        FROM parallel_monitor_lock\n        WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval\n        RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q\n            WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL\n        ) AS workspace_id\n        ",
   "describe": {
     "columns": [
       {
@@ -36,5 +36,5 @@
       null
     ]
   },
-  "hash": "f822df86bf511fe0d3044b69525a6ff2524167877929886adca9b0fa3d3fee45"
+  "hash": "00c4a602aa6a50f2f922851ce63b5216e915c7649698687a00d47da55c70349f"
 }

backend/.sqlx/query-1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE script SET ws_error_handler_muted = $3 WHERE workspace_id = $2 AND path = $1 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f"
-}

backend/.sqlx/query-122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT set_config('session.pgroups', $1, true)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "set_config",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3"
-}

backend/.sqlx/query-1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            gcp_resource_path,\n            subscription_id,\n            topic_id,\n            workspace_id,\n            delivery_type AS \"delivery_type: _\",\n            delivery_config AS \"delivery_config: _\",\n            path,\n            script_path,\n            is_flow,\n            edited_by,\n            email,\n            edited_at,\n            server_id,\n            last_server_ping,\n            extra_perms,\n            error,\n            enabled\n        FROM \n            gcp_trigger\n        WHERE \n            workspace_id = $1 AND \n            path = $2\n        ",
+  "query": "\n        SELECT\n            gcp_resource_path,\n            subscription_id,\n            topic_id,\n            workspace_id,\n            delivery_type AS \"delivery_type: _\",\n            delivery_config AS \"delivery_config: _\",\n            subscription_mode AS \"subscription_mode: _\",\n            path,\n            script_path,\n            is_flow,\n            edited_by,\n            email,\n            edited_at,\n            server_id,\n            last_server_ping,\n            extra_perms,\n            error,\n            enabled\n        FROM \n            gcp_trigger\n        WHERE \n            workspace_id = $1 AND \n            path = $2\n        ",
   "describe": {
     "columns": [
       {
@@ -45,56 +45,71 @@
       },
       {
         "ordinal": 6,
+        "name": "subscription_mode: _",
+        "type_info": {
+          "Custom": {
+            "name": "gcp_subscription_mode",
+            "kind": {
+              "Enum": [
+                "create_update",
+                "existing"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 7,
         "name": "path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "script_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "is_flow",
         "type_info": "Bool"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "extra_perms",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 15,
+        "ordinal": 16,
         "name": "error",
         "type_info": "Text"
       },
       {
-        "ordinal": 16,
+        "ordinal": 17,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -118,6 +133,7 @@
       false,
       false,
       false,
+      false,
       true,
       true,
       false,
@@ -125,5 +141,5 @@
       false
     ]
   },
-  "hash": "f4f6336fc671b00bed7835124892f7a4d3bbe673f7c48153819dab385a5cb357"
+  "hash": "1312b7fd622cc814a406c85dbbff61f003c29185c267642cfd898075ebda855d"
 }

backend/.sqlx/query-18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441.json

@@ -0,0 +1,27 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT set_session_context($1, $2, $3, $4, $5, $6)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "set_session_context",
+        "type_info": "Void"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text",
+        "Text",
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441"
+}

backend/.sqlx/query-26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT jsonb_build_object(\n            'kind', jb.kind,\n            'script_path', jb.runnable_path,\n            'latest_schema', COALESCE(\n                (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.workspace_id = $1 AND s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n                (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.path = jb.runnable_path AND jb.kind = 'flow')\n            ),\n            'schemas', ARRAY(\n                SELECT jsonb_build_object(\n                    'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'),\n                    'job_ids', ARRAY_AGG(DISTINCT j.id),\n                    'schema', (ARRAY_AGG(COALESCE(s.schema, f.schema)))[1]\n                ) FROM v2_job j\n                LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n                LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow'\n                WHERE j.id = ANY(ARRAY_AGG(jb.id))\n                GROUP BY COALESCE(s.hash, f.id)\n            )\n        ) FROM v2_job jb\n        WHERE (jb.kind = 'flow' OR jb.kind = 'script')\n            AND jb.workspace_id = $1 AND jb.id = ANY($2)\n        GROUP BY jb.kind, jb.runnable_path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "jsonb_build_object",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "UuidArray"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "26761fbd7953416eb391de47b1694e0f4ab2bb96a6d838f1b1fdce4b58a8f5d4"
+}

backend/.sqlx/query-29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT set_config('session.folders_read', $1, true)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "set_config",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124"
-}

backend/.sqlx/query-2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))",
+  "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc"
+  "hash": "2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4"
 }

backend/.sqlx/query-2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            SELECT\n                gcp_resource_path,\n                subscription_id,\n                topic_id,\n                workspace_id,\n                delivery_type AS \"delivery_type: _\",\n                delivery_config AS \"delivery_config: _\",\n                path,\n                script_path,\n                is_flow,\n                edited_by,\n                email,\n                edited_at,\n                server_id,\n                last_server_ping,\n                extra_perms,\n                error,\n                enabled\n            FROM\n                gcp_trigger\n            WHERE\n                delivery_type != 'push'::DELIVERY_MODE AND\n                enabled IS TRUE\n                AND (last_server_ping IS NULL OR\n                    last_server_ping < now() - interval '15 seconds'\n                )\n            ",
+  "query": "\n            SELECT\n                gcp_resource_path,\n                subscription_id,\n                topic_id,\n                workspace_id,\n                delivery_type AS \"delivery_type: _\",\n                delivery_config AS \"delivery_config: _\",\n                subscription_mode AS \"subscription_mode: _\",\n                path,\n                script_path,\n                is_flow,\n                edited_by,\n                email,\n                edited_at,\n                server_id,\n                last_server_ping,\n                extra_perms,\n                error,\n                enabled\n            FROM\n                gcp_trigger\n            WHERE\n                delivery_type != 'push'::DELIVERY_MODE AND\n                enabled IS TRUE\n                AND (last_server_ping IS NULL OR\n                    last_server_ping < now() - interval '15 seconds'\n                )\n            ",
   "describe": {
     "columns": [
       {
@@ -45,56 +45,71 @@
       },
       {
         "ordinal": 6,
+        "name": "subscription_mode: _",
+        "type_info": {
+          "Custom": {
+            "name": "gcp_subscription_mode",
+            "kind": {
+              "Enum": [
+                "create_update",
+                "existing"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 7,
         "name": "path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "script_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "is_flow",
         "type_info": "Bool"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "extra_perms",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 15,
+        "ordinal": 16,
         "name": "error",
         "type_info": "Text"
       },
       {
-        "ordinal": 16,
+        "ordinal": 17,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -115,6 +130,7 @@
       false,
       false,
       false,
+      false,
       true,
       true,
       false,
@@ -122,5 +138,5 @@
       false
     ]
   },
-  "hash": "15fbe481789a7817bf37415fb935f9ed537fd7d3b266d928af4d5d3dd8bb5c18"
+  "hash": "2dcff8b16da75740c362015b2293f578a21813f038d75184a5034c37d4daf36e"
 }

backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json

@@ -1,27 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n            VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) RETURNING id AS \"id!\"",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id!",
-        "type_info": "Uuid"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Uuid",
-        "Bool",
-        "Timestamptz",
-        "Varchar",
-        "Int2"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0"
-}

backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval \n    AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
+  "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n    AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
   "describe": {
     "columns": [
       {
@@ -22,5 +22,5 @@
       null
     ]
   },
-  "hash": "ebd25047d14bec8457465ba6ed9b6b15b8f2303157b4c6e80ac9e3d84a57d8b1"
+  "hash": "3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8"
 }

backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS (\n                SELECT 1 FROM v2_as_completed_job \n                WHERE workspace_id = $2 \n                    AND (job_kind = 'appscript' OR job_kind = 'preview')\n                    AND created_by = 'anonymous' \n                    AND started_at > now() - interval '3 hours'\n                    AND script_path LIKE $3 || '/%' \n                    AND result @> ('{\"s3\":\"' || $1 ||  '\"}')::jsonb \n            )",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "exists",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f"
-}

backend/.sqlx/query-443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n                AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND\n                workspace_id = $2)\n                ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827"
+}

backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM metrics WHERE created_at < NOW() - INTERVAL '180 day'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d"
+}

backend/.sqlx/query-5237f9d3f57f4e799968a4a9630c984775d23c7a4dc1a7ad37528a46ab89d6bd.json

@@ -1,80 +1,95 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n        SELECT\n            aws_resource_path,\n            message_attributes,\n            queue_url,\n            workspace_id,\n            path,\n            script_path,\n            is_flow,\n            edited_by,\n            email,\n            edited_at,\n            server_id,\n            last_server_ping,\n            extra_perms,\n            error,\n            enabled\n        FROM \n            sqs_trigger\n        WHERE \n            workspace_id = $1 AND \n            path = $2\n        ",
+  "query": "\n        SELECT\n            aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n            aws_resource_path,\n            message_attributes,\n            queue_url,\n            workspace_id,\n            path,\n            script_path,\n            is_flow,\n            edited_by,\n            email,\n            edited_at,\n            server_id,\n            last_server_ping,\n            extra_perms,\n            error,\n            enabled\n        FROM \n            sqs_trigger\n        WHERE \n            workspace_id = $1 AND \n            path = $2\n        ",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
+        "name": "aws_auth_resource_type: _",
+        "type_info": {
+          "Custom": {
+            "name": "aws_auth_resource_type",
+            "kind": {
+              "Enum": [
+                "oidc",
+                "credentials"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
         "name": "aws_resource_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 1,
+        "ordinal": 2,
         "name": "message_attributes",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 2,
+        "ordinal": 3,
         "name": "queue_url",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 3,
+        "ordinal": 4,
         "name": "workspace_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 4,
+        "ordinal": 5,
         "name": "path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 5,
+        "ordinal": 6,
         "name": "script_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 6,
+        "ordinal": 7,
         "name": "is_flow",
         "type_info": "Bool"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "extra_perms",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "error",
         "type_info": "Text"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -86,6 +101,7 @@
       ]
     },
     "nullable": [
+      false,
       false,
       true,
       false,
@@ -103,5 +119,5 @@
       false
     ]
   },
-  "hash": "2ef82fad8a6ccdc66228cfbce5393de351653ab9ac171fa0eea447c905440867"
+  "hash": "5237f9d3f57f4e799968a4a9630c984775d23c7a4dc1a7ad37528a46ab89d6bd"
 }

backend/.sqlx/query-56264b88a9f428e79c6e531a0011e0dbed634f3b12dedaa5fee0362d670887f6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO app_version\n                (app_id, value, created_by)\n                VALUES ($1, $2::text::json, $3) RETURNING id",
+  "query": "INSERT INTO app_version\n                (app_id, value, created_by, raw_app)\n                VALUES ($1, $2::text::json, $3, $4) RETURNING id",
   "describe": {
     "columns": [
       {
@@ -13,12 +13,13 @@
       "Left": [
         "Int8",
         "Text",
-        "Varchar"
+        "Varchar",
+        "Bool"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "2a3992b5e9abcfbb032d10e142d98efa969dae26a7242eb7ac12593ed5421ef3"
+  "hash": "56264b88a9f428e79c6e531a0011e0dbed634f3b12dedaa5fee0362d670887f6"
 }

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH worker_ids AS (SELECT unnest($1::text[]) as worker)\n            SELECT worker_ids.worker FROM worker_ids\n            LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker\n                WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "worker",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
+}

backend/.sqlx/query-6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT set_config('session.groups', $1, true)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "set_config",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e"
-}

backend/.sqlx/query-6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT set_config('session.user', $1, true)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "set_config",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a"
-}

backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by created_at DESC",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
   "describe": {
     "columns": [
       {
@@ -25,5 +25,5 @@
       true
     ]
   },
-  "hash": "362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675"
+  "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de"
 }

backend/.sqlx/query-776909d3452aaf9e0ed41a5ec314b7bfc5bc8b6a76c98bf4daf7d715d18c52e6.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        INSERT INTO sqs_trigger (\n            aws_auth_resource_type,\n            aws_resource_path,\n            queue_url,\n            message_attributes,\n            workspace_id, \n            path, \n            script_path, \n            is_flow, \n            email, \n            enabled, \n            edited_by\n        ) \n        VALUES (\n            $1, \n            $2, \n            $3, \n            $4, \n            $5, \n            $6, \n            $7,\n            $8,\n            $9,\n            $10,\n            $11\n        )",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        {
+          "Custom": {
+            "name": "aws_auth_resource_type",
+            "kind": {
+              "Enum": [
+                "oidc",
+                "credentials"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Varchar",
+        "Bool",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "776909d3452aaf9e0ed41a5ec314b7bfc5bc8b6a76c98bf4daf7d715d18c52e6"
+}

backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json

@@ -1,21 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n        values ($1, $2, $3, $4, $5, $6, $7, $8) \n        ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Varchar",
-        "Varchar",
-        "Bool",
-        "Bool",
-        "JsonbArray",
-        "TextArray",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f"
-}

backend/.sqlx/query-7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT \n            active_pid \n        FROM \n            pg_replication_slots \n        WHERE \n            slot_name = $1\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "active_pid",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Name"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2"
+}

backend/.sqlx/query-848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE script \n                SET ws_error_handler_muted = $3 \n                WHERE ctid = (\n                    SELECT ctid FROM script\n                    WHERE path = $1 AND workspace_id = $2\n                    ORDER BY created_at DESC\n                    LIMIT 1\n                )\n",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd"
+}

backend/.sqlx/query-8490024b96aa689d3c1bc5cbf94fcd8c5491732818404fee2397f00cccee0ad3.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            UPDATE \n                sqs_trigger \n            SET \n                aws_auth_resource_type = $1,\n                aws_resource_path = $2,\n                queue_url = $3,\n                message_attributes = $4, \n                is_flow = $5, \n                edited_by = $6, \n                email = $7,\n                script_path = $8,\n                path = $9,\n                edited_at = now(), \n                error = NULL,\n                server_id = NULL\n            WHERE \n                workspace_id = $10 AND \n                path = $11\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        {
+          "Custom": {
+            "name": "aws_auth_resource_type",
+            "kind": {
+              "Enum": [
+                "oidc",
+                "credentials"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Bool",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8490024b96aa689d3c1bc5cbf94fcd8c5491732818404fee2397f00cccee0ad3"
+}

backend/.sqlx/query-8eabeee5527da4aad3f46ad185015262b4304667449f3e1a71efd7477a39f7fc.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger\n                 WHERE workspace_id = $1",
+  "query": "\n                SELECT \n                    workspace_id,\n                    path,\n                    url,\n                    script_path,\n                    is_flow,\n                    edited_by,\n                    email,\n                    edited_at,\n                    server_id,\n                    last_server_ping,\n                    extra_perms,\n                    error,\n                    enabled,\n                    filters AS \"filters: _\",\n                    initial_messages AS \"initial_messages: _\",\n                    url_runnable_args AS \"url_runnable_args: _\",\n                    can_return_message\n                FROM \n                    websocket_trigger\n                WHERE \n                    workspace_id = $1\n                ",
   "describe": {
     "columns": [
       {
@@ -114,5 +114,5 @@
       false
     ]
   },
-  "hash": "1aa8ead10f8d994f6685d266fcbd409b0fff43111d9600e64b2348401ed8929d"
+  "hash": "8eabeee5527da4aad3f46ad185015262b4304667449f3e1a71efd7477a39f7fc"
 }

backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
+  "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n             VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n             ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
   "describe": {
     "columns": [],
     "parameters": {
@@ -17,5 +17,5 @@
     },
     "nullable": []
   },
-  "hash": "33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe"
+  "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b"
 }

backend/.sqlx/query-9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT set_config('session.folders_write', $1, true)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "set_config",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21"
-}

backend/.sqlx/query-9b9bc21023f10a0b4bb45879c7da8e119bffd2982b97fc900358506a0a14bbb8.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        INSERT INTO sqs_trigger (\n            aws_resource_path,\n            queue_url,\n            message_attributes,\n            workspace_id, \n            path, \n            script_path, \n            is_flow, \n            email, \n            enabled, \n            edited_by\n        ) \n        VALUES (\n            $1, \n            $2, \n            $3, \n            $4, \n            $5, \n            $6, \n            $7,\n            $8,\n            $9,\n            $10\n        )",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "TextArray",
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Bool",
-        "Varchar",
-        "Bool",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "9b9bc21023f10a0b4bb45879c7da8e119bffd2982b97fc900358506a0a14bbb8"
-}

backend/.sqlx/query-a7df493316f632fc636e4c3c90bef4c98035b5dc808bb8379251c0b35d945ba0.json

@@ -1,80 +1,95 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT * FROM sqs_trigger\n                WHERE workspace_id = $1",
+  "query": "\n                SELECT\n                    aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n                    aws_resource_path,\n                    message_attributes,\n                    queue_url,\n                    workspace_id,\n                    path,\n                    script_path,\n                    is_flow,\n                    edited_by,\n                    email,\n                    edited_at,\n                    server_id,\n                    last_server_ping,\n                    extra_perms,\n                    error,\n                    enabled\n                FROM \n                    sqs_trigger\n                WHERE \n                    workspace_id = $1\n                ",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
+        "name": "aws_auth_resource_type: _",
+        "type_info": {
+          "Custom": {
+            "name": "aws_auth_resource_type",
+            "kind": {
+              "Enum": [
+                "oidc",
+                "credentials"
+              ]
+            }
+          }
+        }
       },
       {
         "ordinal": 1,
-        "name": "queue_url",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
         "name": "aws_resource_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 3,
+        "ordinal": 2,
         "name": "message_attributes",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 4,
-        "name": "script_path",
+        "ordinal": 3,
+        "name": "queue_url",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 5,
-        "name": "is_flow",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 6,
+        "ordinal": 4,
         "name": "workspace_id",
         "type_info": "Varchar"
       },
+      {
+        "ordinal": 5,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
       {
         "ordinal": 7,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 8,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
-      {
-        "ordinal": 10,
-        "name": "extra_perms",
-        "type_info": "Jsonb"
-      },
       {
         "ordinal": 11,
-        "name": "error",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 12,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 13,
+        "ordinal": 12,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
+      {
+        "ordinal": 13,
+        "name": "extra_perms",
+        "type_info": "Jsonb"
+      },
       {
         "ordinal": 14,
+        "name": "error",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 15,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -85,7 +100,6 @@
       ]
     },
     "nullable": [
-      false,
       false,
       false,
       true,
@@ -95,6 +109,8 @@
       false,
       false,
       false,
+      false,
+      false,
       true,
       true,
       true,
@@ -102,5 +118,5 @@
       false
     ]
   },
-  "hash": "2b6c13191484b0c664f35e2c811082b00c44fd4a7e98a11b714133674a7b6da7"
+  "hash": "a7df493316f632fc636e4c3c90bef4c98035b5dc808bb8379251c0b35d945ba0"
 }

backend/.sqlx/query-a9f6e5720f748418ae52cf0e31778b9bbef26008803f2dd2a043bdb091b2ff20.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO app_version\n            (app_id, value, created_by)\n            VALUES ($1, $2::text::json, $3) RETURNING id",
+  "query": "INSERT INTO app_version\n            (app_id, value, created_by, raw_app)\n            VALUES ($1, $2::text::json, $3, $4) RETURNING id",
   "describe": {
     "columns": [
       {
@@ -13,12 +13,13 @@
       "Left": [
         "Int8",
         "Text",
-        "Varchar"
+        "Varchar",
+        "Bool"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "83d18f4e1cda2c1867168551d855d5626e766b5469c6976c986bdafc8b9407c5"
+  "hash": "a9f6e5720f748418ae52cf0e31778b9bbef26008803f2dd2a043bdb091b2ff20"
 }

backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                    SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND\n                    workspace_id = $2)\n                    ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "content",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145"
-}

backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json

@@ -0,0 +1,85 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT \n                    j.id,\n                    j.kind AS \"kind: _\",\n                    COALESCE(s.path, f.path) AS \"script_path!\",\n                    COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n                    COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n                    args AS input,\n                    COALESCE(s.schema, f.schema) AS \"schema: _\"\n                FROM v2_job j\n                LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n                LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n                LEFT JOIN v2_job_completed jc ON jc.id = j.id\n                LEFT JOIN v2_job_queue jq ON jq.id = j.id\n                WHERE j.id = ANY($1)\n                    AND j.workspace_id = $2\n                    AND COALESCE(s.hash, f.id) IS NOT NULL\n                    AND COALESCE(s.path, f.path) IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 1,
+        "name": "kind: _",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "script_path!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "script_hash!: _",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 4,
+        "name": "scheduled_for!: _",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 5,
+        "name": "input",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 6,
+        "name": "schema: _",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      null,
+      null,
+      null,
+      true,
+      null
+    ]
+  },
+  "hash": "ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9"
+}

backend/.sqlx/query-b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND workspace_id = $2)",
+  "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       false
     ]
   },
-  "hash": "a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb"
+  "hash": "b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac"
 }

backend/.sqlx/query-bc0bedddcafad216c30c5061eef3f4ed8573cda89f655d6490d22f182e3f2f36.json

@@ -1,80 +1,95 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            SELECT\n                queue_url,\n                aws_resource_path,\n                message_attributes,\n                workspace_id,\n                path,\n                script_path,\n                is_flow,\n                edited_by,\n                email,\n                edited_at,\n                server_id,\n                last_server_ping,\n                extra_perms,\n                error,\n                enabled\n            FROM\n                sqs_trigger\n            WHERE\n                enabled IS TRUE\n                AND (last_server_ping IS NULL OR\n                    last_server_ping < now() - interval '15 seconds'\n                )\n            ",
+  "query": "\n            SELECT\n                aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n                queue_url,\n                aws_resource_path,\n                message_attributes,\n                workspace_id,\n                path,\n                script_path,\n                is_flow,\n                edited_by,\n                email,\n                edited_at,\n                server_id,\n                last_server_ping,\n                extra_perms,\n                error,\n                enabled\n            FROM\n                sqs_trigger\n            WHERE\n                enabled IS TRUE\n                AND (last_server_ping IS NULL OR\n                    last_server_ping < now() - interval '15 seconds'\n                )\n            ",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
+        "name": "aws_auth_resource_type: _",
+        "type_info": {
+          "Custom": {
+            "name": "aws_auth_resource_type",
+            "kind": {
+              "Enum": [
+                "oidc",
+                "credentials"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
         "name": "queue_url",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 1,
+        "ordinal": 2,
         "name": "aws_resource_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 2,
+        "ordinal": 3,
         "name": "message_attributes",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 3,
+        "ordinal": 4,
         "name": "workspace_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 4,
+        "ordinal": 5,
         "name": "path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 5,
+        "ordinal": 6,
         "name": "script_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 6,
+        "ordinal": 7,
         "name": "is_flow",
         "type_info": "Bool"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "extra_perms",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "error",
         "type_info": "Text"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -83,6 +98,7 @@
       "Left": []
     },
     "nullable": [
+      false,
       false,
       false,
       true,
@@ -100,5 +116,5 @@
       false
     ]
   },
-  "hash": "e6adaebcade2e25be800e8b888b23c94caae2421f3cae7c06d6346bd6de1d94a"
+  "hash": "bc0bedddcafad216c30c5061eef3f4ed8573cda89f655d6490d22f182e3f2f36"
 }

backend/.sqlx/query-c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                SELECT\n                    gcp_resource_path,\n                    subscription_id,\n                    topic_id,\n                    workspace_id,\n                    delivery_type AS \"delivery_type: _\",\n                    delivery_config AS \"delivery_config: _\",\n                    path,\n                    script_path,\n                    is_flow,\n                    edited_by,\n                    email,\n                    edited_at,\n                    server_id,\n                    last_server_ping,\n                    extra_perms,\n                    error,\n                    enabled\n                FROM \n                    gcp_trigger\n                WHERE \n                    workspace_id = $1\n                ",
+  "query": "\n                SELECT\n                    gcp_resource_path,\n                    subscription_id,\n                    topic_id,\n                    workspace_id,\n                    delivery_type AS \"delivery_type: _\",\n                    delivery_config AS \"delivery_config: _\",\n                    subscription_mode AS \"subscription_mode: _\",\n                    path,\n                    script_path,\n                    is_flow,\n                    edited_by,\n                    email,\n                    edited_at,\n                    server_id,\n                    last_server_ping,\n                    extra_perms,\n                    error,\n                    enabled\n                FROM \n                    gcp_trigger\n                WHERE \n                    workspace_id = $1\n                ",
   "describe": {
     "columns": [
       {
@@ -45,56 +45,71 @@
       },
       {
         "ordinal": 6,
+        "name": "subscription_mode: _",
+        "type_info": {
+          "Custom": {
+            "name": "gcp_subscription_mode",
+            "kind": {
+              "Enum": [
+                "create_update",
+                "existing"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 7,
         "name": "path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "script_path",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "is_flow",
         "type_info": "Bool"
       },
       {
-        "ordinal": 9,
+        "ordinal": 10,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "email",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "edited_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "server_id",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "last_server_ping",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "extra_perms",
         "type_info": "Jsonb"
       },
       {
-        "ordinal": 15,
+        "ordinal": 16,
         "name": "error",
         "type_info": "Text"
       },
       {
-        "ordinal": 16,
+        "ordinal": 17,
         "name": "enabled",
         "type_info": "Bool"
       }
@@ -117,6 +132,7 @@
       false,
       false,
       false,
+      false,
       true,
       true,
       false,
@@ -124,5 +140,5 @@
       false
     ]
   },
-  "hash": "0102308ffa1c0dbfba54d29246535bb81146a4cfae0ec408435570e0813a3bef"
+  "hash": "c2925a28212265bd9ac8e2d498b3588fc040c7a29ae6d9bab1f05af0b535e2ac"
 }

backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741"
-}

backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n            created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n            script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n            flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n            cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $21, $22, $23, $24, $25, $26,\n            CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)",
+  "query": "WITH inserted_job AS (\n            INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n                created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n            script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n            flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n            cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $21, $22, $23, $24, $25, $26,\n            CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n        ),\n        inserted_runtime AS (\n            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n        ),\n        inserted_job_perms AS (\n            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n            values ($1, $32, $33, $34, $35, $36, $37, $2) \n            ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2\n        )\n        INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 THEN now() END, $30, $31)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -86,10 +86,20 @@
         "Varchar",
         "Int4",
         "Int2",
-        "Bool"
+        "Bool",
+        "Bool",
+        "Timestamptz",
+        "Varchar",
+        "Int2",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "JsonbArray",
+        "TextArray"
       ]
     },
     "nullable": []
   },
-  "hash": "29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e"
+  "hash": "cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6"
 }

backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "deployment_msg",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec"
+}

backend/.sqlx/query-d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COALESCE(\n                (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n                (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow')\n            ) FROM v2_job jb\n            WHERE jb.id = $1 AND jb.workspace_id = $2\n            GROUP BY jb.kind, jb.runnable_path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "coalesce",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2"
+}

backend/.sqlx/query-daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH to_update AS (\n                SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n                FROM v2_job_queue q\n                JOIN v2_job j ON j.id = q.id\n                JOIN v2_job_runtime r ON r.id = j.id\n                LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n                WHERE ping < now() - ($1 || ' seconds')::interval\n                    AND running = true\n                    AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n                    AND same_worker = false\n                    AND (zjc.counter IS NULL OR zjc.counter <= $2)\n                FOR UPDATE of q SKIP LOCKED\n            ),\n            zombie_jobs AS (\n                UPDATE v2_job_queue q\n                SET running = false, started_at = null\n                FROM to_update tu\n                WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n                RETURNING q.id, q.workspace_id, ping, tu.counter\n            ),\n            update_ping AS (\n                UPDATE v2_job_runtime r\n                SET ping = null\n                FROM zombie_jobs zj\n                WHERE r.id = zj.id\n            ),\n            increment_counter AS (\n                INSERT INTO zombie_job_counter (job_id, counter)\n                SELECT id, 1 FROM to_update WHERE counter < $2\n                ON CONFLICT (job_id) DO UPDATE \n                SET counter = zombie_job_counter.counter + 1\n            ),\n            update_concurrency AS (\n                UPDATE concurrency_counter cc\n                SET job_uuids = job_uuids - zj.id::text\n                FROM zombie_jobs zj\n                INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n                WHERE cc.concurrency_id = ck.key\n            )\n            SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
+  "query": "WITH to_update AS (\n                SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n                FROM v2_job_queue q\n                JOIN v2_job j ON j.id = q.id\n                JOIN v2_job_runtime r ON r.id = j.id\n                LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n                WHERE ping < now() - ($1 || ' seconds')::interval\n                    AND running = true\n                    AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n                    AND same_worker = false\n                    AND (zjc.counter IS NULL OR zjc.counter <= $2)\n                FOR UPDATE of q SKIP LOCKED\n            ),\n            zombie_jobs AS (\n                UPDATE v2_job_queue q\n                SET running = false, started_at = null\n                FROM to_update tu\n                WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n                RETURNING q.id, q.workspace_id, ping, tu.counter\n            ),\n            update_ping AS (\n                UPDATE v2_job_runtime r\n                SET ping = null\n                FROM zombie_jobs zj\n                WHERE r.id = zj.id\n            ),\n            increment_counter AS (\n                INSERT INTO zombie_job_counter (job_id, counter)\n                SELECT id, 1 FROM to_update WHERE counter < $2\n                ON CONFLICT (job_id) DO UPDATE\n                SET counter = zombie_job_counter.counter + 1\n            ),\n            update_concurrency AS (\n                UPDATE concurrency_counter cc\n                SET job_uuids = job_uuids - zj.id::text\n                FROM zombie_jobs zj\n                INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n                WHERE cc.concurrency_id = ck.key\n            )\n            SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
   "describe": {
     "columns": [
       {
@@ -37,5 +37,5 @@
       null
     ]
   },
-  "hash": "abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5"
+  "hash": "daf9674838fb3e3653a356c7434c719616a614d77e726433737e5f5d9bd60134"
 }

backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) \n            SELECT worker_ids.worker FROM worker_ids \n            LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker \n                WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "worker",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "TextArray"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927"
-}

backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS (\n                SELECT 1 FROM v2_as_completed_job\n                WHERE workspace_id = $2\n                    AND (job_kind = 'appscript' OR job_kind = 'preview')\n                    AND created_by = 'anonymous'\n                    AND started_at > now() - interval '3 hours'\n                    AND script_path LIKE $3 || '/%'\n                    AND result @> ('{\"s3\":\"' || $1 ||  '\"}')::jsonb\n            )",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b"
+}

backend/.sqlx/query-f68d23841e0e31cdf8633aaf0f32777e04e4965682e5b37afbe84194b756d5f5.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE \n                sqs_trigger \n            SET \n                aws_resource_path = $1,\n                queue_url = $2,\n                message_attributes = $3, \n                is_flow = $4, \n                edited_by = $5, \n                email = $6,\n                script_path = $7,\n                path = $8,\n                edited_at = now(), \n                error = NULL,\n                server_id = NULL\n            WHERE \n                workspace_id = $9 AND \n                path = $10\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "TextArray",
-        "Bool",
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "f68d23841e0e31cdf8633aaf0f32777e04e4965682e5b37afbe84194b756d5f5"
-}

backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json

@@ -0,0 +1,21 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n        SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "UuidArray",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "JsonbArray",
+        "TextArray",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa"
+}

backend/.vscode/settings.json

@@ -11,5 +11,5 @@
   "remote.autoForwardPorts": true,
   "conventionalCommits.scopes": [
     "restructring triggers, decoding trigger message on work"
-  ],
+  ]
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.481.0"
+version = "1.486.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.481.0"
+version = "1.486.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -64,6 +64,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
 tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
 sqlx = ["windmill-worker/sqlx"]
+deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"]
 kafka = ["windmill-api/kafka"]
 nats = ["windmill-api/nats"]
 otel = ["windmill-common/otel", "windmill-worker/otel"]
@@ -71,8 +72,9 @@ dind = ["windmill-worker/dind"]
 websocket = ["windmill-api/websocket"]
 http_trigger = ["windmill-api/http_trigger"]
 postgres_trigger = ["windmill-api/postgres_trigger"]
+mcp = ["windmill-api/mcp"]
 mqtt_trigger = ["windmill-api/mqtt_trigger"]
-sqs_trigger = ["windmill-api/sqs_trigger"]
+sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"]
 gcp_trigger = ["windmill-api/gcp_trigger"]
 smtp = ["windmill-api/smtp", "windmill-common/smtp"]
 license = ["windmill-api/license"]
@@ -82,7 +84,6 @@ static_frontend = ["windmill-api/static_frontend"]
 scoped_cache = ["windmill-common/scoped_cache"]
 # Languages
 python = ["windmill-worker/python"]
-deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
 rust = ["windmill-worker/rust"]
 mysql = ["windmill-worker/mysql"]
 oracledb = ["windmill-worker/oracledb"]
@@ -98,6 +99,7 @@ all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "
 [dependencies]
 anyhow.workspace = true
 tokio.workspace = true
+tokio-stream.workspace = true
 dotenv.workspace = true
 windmill-queue.workspace = true
 windmill-common = { workspace = true, default-features = false }
@@ -130,6 +132,8 @@ quote.workspace = true
 memchr.workspace = true
 v8 = { workspace = true, optional = true }
 rustls.workspace = true
+systemstat.workspace = true
+size.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = { optional = true, workspace = true }
@@ -181,6 +185,7 @@ axum = { version = "^0.7", features = ["multipart"] }
 headers = "^0"
 hyper = { version = "^1", features = ["full"] }
 tokio = { version = "^1.42.0", features = ["full", "tracing", "time"] }
+tokio-stream = { version = "0.1.17" }
 tower = "^0"
 tower-http = { version = "^0.6", features = ["trace", "cors"] }
 tower-cookies = "^0.10"
@@ -189,7 +194,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^2"
 anyhow = "^1"
-chrono = { version = "=0.4.39", features = ["serde"] }
+chrono = { version = "^0.4", features = ["serde"] }
 chrono-tz = "^0.10.1"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
@@ -302,6 +307,7 @@ postgres-native-tls = "^0"
 native-tls = "^0"
 # samael will break compilation on MacOS. Use this fork instead to make it work
 # samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
+libxml = { version = "=0.3.3" }
 samael = { version="0.0.14", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
 rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]}
@@ -311,9 +317,9 @@ nix = { version = "0.27.1", features = ["process", "signal"] }
 tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" }
 hf-hub = "0.3.2"
 tokenizers = "0.14.1"
-candle-core = "0.3.0"
-candle-transformers = "0.3.0"
-candle-nn = "0.3.0"
+candle-core = "0.9.1"
+candle-transformers = "0.9.1"
+candle-nn = "0.9.1"
 tiberius = { version = "0.12.3", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]}
 pin-project = "1"
 indexmap = { version = "2.2.5", features = ["serde"]}
@@ -327,8 +333,8 @@ async-nats = "0.38.0"
 nkeys = "0.4.4"
 nu-parser = { version = "0.101.0", default-features = false }
 
-datafusion = "39.0.0"
-object_store = { version = "0.10.0", features = ["aws", "azure"] }
+datafusion = "47.0.0"
+object_store = { version = "0.12.0", features = ["aws", "azure"] }
 openidconnect = { version = "4.0.0-rc.1" }
 aws-config = "^1"
 aws-sdk-sqs = "1.57.0"
@@ -351,9 +357,6 @@ bollard = "0.18.1"
 tonic = { version = "=0.12.3", features = ["tls-native-roots"] }
 byteorder = "1.5.0"
 
-# todo remove
-half = "=2.4.1"
-
 tikv-jemallocator = { version = "0.5" }
 tikv-jemalloc-sys  = { version = "^0.5" }
 tikv-jemalloc-ctl = { version = "^0.5" }
@@ -364,6 +367,10 @@ pin-project-lite = "^0"
 tantivy = "0.22.0"
 
 backon = "1.3.0"
+systemstat = "0.2.4"
+size = "0.5.0"
+
+flume = { version = "0.11.1", features = ["async"] }
 
 # Macro-related
 proc-macro2 = "1.0"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-85c37983ffb8f622458425c182613206625c6cee
+868ccad87afb804fe22818ecd3d5a091199bcdbf

backend/migrations/20241223155748_raw_apps_v2.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE app_version DROP COLUMN IF EXISTS raw_app;

backend/migrations/20241223155748_raw_apps_v2.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE app_version ADD COLUMN IF NOT EXISTS raw_app BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20250412144540_improve_perf_api_role.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250412144540_improve_perf_api_role.up.sql

@@ -0,0 +1,22 @@
+-- Add up migration script here
+ CREATE OR REPLACE FUNCTION set_session_context(
+    admin BOOLEAN,
+    username TEXT,
+    groups TEXT,
+    pgroups TEXT,
+    folders_read TEXT,
+    folders_write TEXT
+) RETURNS void AS $$
+BEGIN
+    IF admin THEN
+        SET LOCAL ROLE windmill_admin;
+    ELSE
+        SET LOCAL ROLE windmill_user;
+    END IF;
+    PERFORM set_config('session.user', username, true);
+    PERFORM set_config('session.groups', groups, true);
+    PERFORM set_config('session.pgroups', pgroups, true);
+    PERFORM set_config('session.folders_read', folders_read, true);
+    PERFORM set_config('session.folders_write', folders_write, true);
+END;
+$$ LANGUAGE plpgsql;

backend/migrations/20250417132646_add-resource-type-column-to-sqs.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE sqs_trigger DROP COLUMN aws_auth_resource_type;
+DROP TYPE IF EXISTS AWS_AUTH_RESOURCE_TYPE;

backend/migrations/20250417132646_add-resource-type-column-to-sqs.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+CREATE TYPE AWS_AUTH_RESOURCE_TYPE AS ENUM ('oidc', 'credentials');
+ALTER TABLE sqs_trigger 
+    ADD COLUMN aws_auth_resource_type AWS_AUTH_RESOURCE_TYPE DEFAULT 'credentials'::AWS_AUTH_RESOURCE_TYPE NOT NULL;

backend/migrations/20250420144035_add-new-column-to-gcp-trigger.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE gcp_trigger DROP COLUMN subscription_mode;
+DROP TYPE GCP_SUBSCRIPTION_MODE;

backend/migrations/20250420144035_add-new-column-to-gcp-trigger.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+CREATE TYPE GCP_SUBSCRIPTION_MODE AS ENUM ('create_update', 'existing');
+ALTER TABLE gcp_trigger ADD COLUMN subscription_mode GCP_SUBSCRIPTION_MODE NOT NULL DEFAULT 'create_update'::GCP_SUBSCRIPTION_MODE;

backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE websocket_trigger
+ALTER COLUMN url TYPE VARCHAR(255);

backend/migrations/20250421120705_alter_url_column_length_on_websocket_triggers.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE websocket_trigger
+ALTER COLUMN url TYPE VARCHAR(1000);

backend/migrations/20250424144434_critical_alerts_on_db_oversize.down.sql

@@ -0,0 +1 @@
+DELETE FROM global_settings WHERE name = 'critical_alerts_on_db_oversize';

backend/migrations/20250424144434_critical_alerts_on_db_oversize.up.sql

@@ -0,0 +1,3 @@
+INSERT INTO global_settings (name, value) 
+VALUES ('critical_alerts_on_db_oversize', '{}') 
+ON CONFLICT (name) DO NOTHING;

backend/migrations/20250428170426_mcp_mode_log.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250428170426_mcp_mode_log.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE log_mode ADD VALUE 'mcp';

backend/migrations/20250429211554_create_indices_on_queue.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250429211554_create_indices_on_queue.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS idx_metrics_id_created_at ON public.metrics (id, created_at DESC) WHERE id LIKE 'queue_%';

backend/migrations/20250429214657_create_indices_on_job_stats.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250429214657_create_indices_on_job_stats.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS job_stats_id ON job_stats (job_id);

backend/migrations/20250506151818_orderby_refactor.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250506151818_orderby_refactor.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+DROP INDEX IF EXISTS index_script_on_path_created_at;
+CREATE INDEX IF NOT EXISTS index_script_on_path_created_at ON script (workspace_id, path, created_at DESC);

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -11,6 +11,7 @@ mod mapping;
 use async_recursion::async_recursion;
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use std::collections::HashMap;
 
 use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
 #[cfg(not(target_arch = "wasm32"))]
@@ -20,6 +21,7 @@ use regex_lite::Regex;
 
 use rustpython_parser::{
     ast::{Stmt, StmtImport, StmtImportFrom, Suite},
+    text_size::TextRange,
     Parse,
 };
 use sqlx::{Pool, Postgres};
@@ -41,9 +43,11 @@ fn replace_full_import(x: &str) -> Option<String> {
 
 lazy_static! {
     static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap();
+    static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap();
+    static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap();
 }
 
-fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<String> {
+fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<NImport> {
     if level > 0 {
         let mut imports = vec![];
         let splitted_path = path.split("/");
@@ -52,17 +56,18 @@ fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<Strin
             .take(splitted_path.count() - level)
             .join("/");
         if let Some(m) = module {
-            imports.push(format!("relative:{base}/{}", m.replace(".", "/")));
+            imports.push(NImport::Relative(format!("{base}/{}", m.replace(".", "/"))));
         } else {
-            imports.push(format!("relative:{base}"));
+            imports.push(NImport::Relative(format!("{base}")));
         }
         imports
     } else if let Some(module) = module {
         let imprt = module.split('.').next().unwrap_or("").replace("_", "-");
         if imprt == "u" || imprt == "f" {
-            vec![format!("relative:{}", module.replace(".", "/"))]
+            vec![NImport::Relative(module.replace(".", "/"))]
         } else {
-            vec![replace_full_import(&module).unwrap_or(replace_import(imprt))]
+            let pkg = replace_full_import(&module).unwrap_or(replace_import(imprt));
+            vec![NImport::Auto { key: if module == pkg { None } else { Some(module) }, pkg }]
         }
     } else {
         vec![]
@@ -73,17 +78,68 @@ pub fn parse_relative_imports(code: &str, path: &str) -> error::Result<Vec<Strin
     let nimports = parse_code_for_imports(code, path)?;
     return Ok(nimports
         .into_iter()
-        .filter_map(|x| {
-            if x.starts_with("relative:") {
-                Some(x.replace("relative:", ""))
-            } else {
-                None
-            }
+        .filter_map(|x| match x {
+            NImport::Relative(path) => Some(path),
+            _ => None,
         })
         .collect());
 }
 
-fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>> {
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+enum NImport {
+    // Order matters! First we want to resolve all repins
+
+    // manually repinned requirement
+    // e.g.:
+    // import pandas # repin: pandas==x.y.z
+    Repin {
+        pin: ImportPin,
+        key: String,
+    },
+    // manually pinned requirements
+    // e.g.:
+    // import pandas # pin: pandas>=x.y.z
+    // import pandas # pin: pandas<=x.y.z
+    //
+    // NOTE: It is possible for multiple pins exist on same import
+    // That's why we store vector of pins
+    Pin {
+        pins: Vec<ImportPin>,
+        key: String,
+    },
+    // Automatically inferred requirement
+    // e.g.:
+    // import pandas
+    Auto {
+        // Take `x.y.z` for example
+        // x is going to be the `root`
+        // and x.y.z is `full`
+        //
+        // `full` will be None if it is equal to root
+        //
+        // We will use `root` as a requirement name and pass to `uv pip compile` if it was not replaced with any pin
+        pkg: String,
+
+        // However we still need full, since all pins pin against full import names
+        key: Option<String>,
+    },
+    // Relative imports
+    Relative(String),
+}
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+enum NImportResolved {
+    Repin { pin: ImportPin, key: String },
+    Pin { pins: Vec<ImportPin>, key: String },
+    Auto { pkg: String, key: Option<String> },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+struct ImportPin {
+    pkg: String,
+    path: String,
+}
+
+fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<NImport>> {
     let mut code = code.split(DEF_MAIN).next().unwrap_or("").to_string();
 
     // remove main function decorator from end of file if it exists
@@ -104,19 +160,54 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>>
     let ast = Suite::parse(&code, "main.py").map_err(|e| {
         error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string()))
     })?;
-    let nimports: Vec<String> = ast
+
+    let find_pin = |range: TextRange, key: String| {
+        let hs = code
+            .chars()
+            .skip(range.end().to_usize())
+            .take_while(|e| *e != '\n')
+            .collect::<String>();
+
+        if hs.trim_start().is_empty() {
+            return None;
+        }
+
+        PIN_RE.captures(&hs).and_then(|x| {
+            x.get(1).zip(x.get(2)).and_then(|(ty_m, pkg_m)| {
+                let pkg = pkg_m.as_str().to_owned();
+                if ty_m.as_str() == "pin" {
+                    Some(vec![NImport::Pin {
+                        pins: vec![ImportPin { pkg, path: path.to_owned() }],
+                        key,
+                    }])
+                } else if ty_m.as_str() == "repin" {
+                    Some(vec![NImport::Repin {
+                        pin: ImportPin { pkg, path: path.to_owned() },
+                        key,
+                    }])
+                } else {
+                    None
+                }
+            })
+        })
+    };
+
+    let mut nimports: Vec<NImport> = ast
         .into_iter()
         .filter_map(|x| match x {
-            Stmt::Import(StmtImport { names, .. }) => Some(
-                names
-                    .into_iter()
-                    .map(|x| {
-                        let name = x.name.to_string();
-                        process_import(Some(name), path, 0)
-                    })
-                    .flatten()
-                    .collect::<Vec<String>>(),
-            ),
+            Stmt::Import(StmtImport { names, range }) => names
+                .get(0)
+                .and_then(|al| find_pin(range, al.name.to_string()))
+                .or(Some(
+                    names
+                        .into_iter()
+                        .map(|x| {
+                            let name = x.name.to_string();
+                            process_import(Some(name), path, 0)
+                        })
+                        .flatten()
+                        .collect::<Vec<NImport>>(),
+                )),
             Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
                 Some(process_import(
                     module.map(|x| x.to_string()),
@@ -124,15 +215,25 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>>
                     i.to_usize(),
                 ))
             }
-            Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
-                Some(process_import(module.map(|x| x.to_string()), path, 0))
-            }
+            Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin(
+                range,
+                module.clone().map(|x| x.to_string()).unwrap_or_default(),
+            )
+            .or(Some(process_import(module.map(|x| x.to_string()), path, 0))),
             _ => None,
         })
         .flatten()
-        .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+        .filter(|x| {
+            if let NImport::Auto { ref pkg, .. } = x {
+                !STDIMPORTS.contains(&(*pkg).as_str())
+            } else {
+                true
+            }
+        })
         .unique()
         .collect();
+
+    nimports.sort();
     return Ok(nimports);
 }
 
@@ -143,8 +244,9 @@ pub async fn parse_python_imports(
     db: &Pool<Postgres>,
     already_visited: &mut Vec<String>,
     annotated_pyv_numeric: &mut Option<u32>,
-) -> error::Result<Vec<String>> {
-    parse_python_imports_inner(
+) -> error::Result<(Vec<String>, Option<String>)> {
+    let mut compile_error_hint: Option<String> = None;
+    let mut imports = parse_python_imports_inner(
         code,
         w_id,
         path,
@@ -153,7 +255,46 @@ pub async fn parse_python_imports(
         annotated_pyv_numeric,
         &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())),
     )
-    .await
+    .await?
+    .into_values()
+    .map(|nimport| match nimport {
+        NImportResolved::Pin { pins, .. } => pins.into_iter().map(|p| {
+            if let Some(hint) = &mut compile_error_hint{
+                    hint.push_str(&format!("\n - pin to {} in {}", p.pkg, p.path));
+            } else {
+                compile_error_hint = Some("\n\nMultiple pins can cause problems during lockfile resolution.\nMake sure you checked every pin for conflicts:\n".into())
+            };
+            Ok(p.pkg)
+        }).collect_vec(),
+        NImportResolved::Repin { pin: ImportPin { pkg, .. }, .. } => vec![Ok(pkg)],
+        NImportResolved::Auto { pkg, key } => vec![
+
+            if let Some(key) = key {
+               Ok(format!("{pkg} # (mapped from {key})"))
+            } else {
+               Ok(pkg)
+            }
+        ],
+    })
+    .flatten()
+    .collect::<error::Result<Vec<String>>>()?
+    .into_iter()
+    .unique()
+    .collect_vec();
+
+    imports.sort();
+
+    compile_error_hint
+        .as_mut()
+        .map(|e| e.push_str("\n\nNOTE: You can also `repin` to override all pins"));
+    Ok((imports, compile_error_hint))
+}
+
+fn extract_pkg_name(requirement: &str) -> String {
+    PKG_RE
+        .captures(requirement)
+        .map(|x| x.get(1).map(|m| m.as_str().to_string()).unwrap_or_default())
+        .unwrap_or_default()
 }
 
 #[async_recursion]
@@ -165,7 +306,7 @@ async fn parse_python_imports_inner(
     already_visited: &mut Vec<String>,
     annotated_pyv_numeric: &mut Option<u32>,
     path_where_annotated_pyv: &mut Option<String>,
-) -> error::Result<Vec<String>> {
+) -> error::Result<HashMap<String, NImportResolved>> {
     let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
 
     // we pass only if there is none or only one annotation
@@ -194,7 +335,6 @@ async fn parse_python_imports_inner(
             } else {
                 *annotated_pyv_numeric = Some(numeric);
             }
-
             *path_where_annotated_pyv = Some(path.to_owned());
         }
         Ok(())
@@ -209,74 +349,216 @@ async fn parse_python_imports_inner(
         .lines()
         .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
     if let Some((pos, _)) = find_requirements {
-        let lines = code
-            .lines()
+        let mut requirements = HashMap::new();
+        code.lines()
             .skip(pos + 1)
             .map_while(|x| {
-                RE.captures(x)
-                    .map(|x| x.get(1).unwrap().as_str().to_string())
+                RE.captures(x).and_then(|x| {
+                    x.get(1).map(|m| {
+                        let requirement = m.as_str().to_string();
+                        let key = extract_pkg_name(&requirement);
+                        requirements.insert(
+                            key.clone(),
+                            NImportResolved::Pin {
+                                pins: vec![ImportPin {
+                                    pkg: requirement.clone(),
+                                    path: Default::default(),
+                                }],
+                                key,
+                            },
+                        );
+                    })
+                })
             })
-            .collect();
-        Ok(lines)
+            .collect_vec();
+
+        Ok(requirements)
     } else {
         let find_extra_requirements = code.lines().find_position(|x| {
             x.starts_with("#extra_requirements:") || x.starts_with("# extra_requirements:")
         });
-        let mut imports: Vec<String> = vec![];
+        let mut imports: HashMap<String, NImportResolved> = HashMap::new();
         if let Some((pos, _)) = find_extra_requirements {
-            let lines: Vec<String> = code
-                .lines()
+            code.lines()
                 .skip(pos + 1)
                 .map_while(|x| {
-                    RE.captures(x)
-                        .map(|x| x.get(1).unwrap().as_str().to_string())
+                    RE.captures(x).and_then(|x| {
+                        x.get(1).map(|m| {
+                            let requirement = m.as_str().to_string();
+                            let key = extract_pkg_name(&requirement);
+                            imports.insert(
+                                key.clone(),
+                                NImportResolved::Pin {
+                                    pins: vec![ImportPin {
+                                        pkg: requirement,
+                                        path: Default::default(),
+                                    }],
+                                    key,
+                                },
+                            );
+                        })
+                    })
                 })
-                .collect();
-            imports.extend(lines);
+                .collect_vec();
         }
 
-        let nimports = parse_code_for_imports(code, path)?;
-        for n in nimports.iter() {
-            let nested = if n.starts_with("relative:") {
-                let rpath = n.replace("relative:", "");
-                let code = sqlx::query_scalar!(
-                    r#"
-                    SELECT content FROM script WHERE path = $1 AND workspace_id = $2
-                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND
-                    workspace_id = $2)
-                    "#,
-                    &rpath,
-                    w_id
-                )
-                .fetch_optional(db)
-                .await?
-                .unwrap_or_else(|| "".to_string());
+        // Will get unsorted vector of imports found in current script
+        let mut nimports = parse_code_for_imports(code, path)?;
 
-                if already_visited.contains(&rpath) {
-                    vec![]
-                } else {
-                    already_visited.push(rpath.clone());
-                    parse_python_imports_inner(
-                        &code,
-                        w_id,
+        // It is important to note, that sorting is important and will always result in this pattern:
+        // 1. All Repins go first
+        // 2. All Pins go second
+        // 3. All Auto go third
+        // 4. All relative imports go the last
+        //
+        // This way we make sure all repins are resolved before (re)pins inside imported relative scripts.
+        nimports.sort();
+
+        for n in nimports.into_iter() {
+            let mut nested = match n {
+                NImport::Relative(rpath) => {
+                    let code = sqlx::query_scalar!(
+                        r#"
+                SELECT content FROM script WHERE path = $1 AND workspace_id = $2
+                AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND
+                workspace_id = $2)
+                "#,
                         &rpath,
-                        db,
-                        already_visited,
-                        annotated_pyv_numeric,
-                        path_where_annotated_pyv,
+                        w_id
                     )
+                    .fetch_optional(db)
                     .await?
+                    .unwrap_or_else(|| "".to_string());
+
+                    if already_visited.contains(&rpath) {
+                        vec![]
+                    } else {
+                        already_visited.push(rpath.clone());
+                        // Because the algo goes depth first, this function will never return relative import
+                        // This why we can safely assume later, that there is no relative imports
+                        parse_python_imports_inner(
+                            &code,
+                            w_id,
+                            &rpath,
+                            db,
+                            already_visited,
+                            annotated_pyv_numeric,
+                            path_where_annotated_pyv,
+                        )
+                        .await?
+                        .into_values()
+                        .collect_vec()
+                    }
                 }
-            } else {
-                vec![n.to_string()]
+                NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }],
+                NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }],
+                NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }],
             };
+
+            // Nested should also be sorted for the same reason
+            nested.sort();
+
+            // At this point there should be no NImport::Relative in `nested`
             for imp in nested {
-                if !imports.contains(&imp) {
-                    imports.push(imp);
+                let key = match imp.clone() {
+                    NImportResolved::Pin { key, .. } => key,
+                    NImportResolved::Repin { key, .. } => key,
+                    NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg),
+                };
+                // Handled cases:
+                //
+                //  1.
+                //  Error: Imported windmill scripts have different pins
+                //
+                //  auto
+                //  ├── pin:2
+                //  └── pin:1
+                //
+                //  Fix 1:
+                //
+                //  auto
+                //  ├── pin:1
+                //  └── pin:1
+                //
+                //  Fix 2:
+                //
+                //  repin:1
+                //  ├── pin:2
+                //  └── pin:1
+                //
+                //  2.
+                //  Error: Imported windmill scripts have different pins
+                //
+                //  pin:2
+                //  └── pin:1
+                //
+                //  Fix 1:
+                //
+                //  auto
+                //  └── pin:1
+                //
+                //  Fix 2:
+                //
+                //  repin:2
+                //  └── pin:1
+                //
+                //  3. repins allowed to be repinned again
+                //
+                //  repin:2
+                //  └── repin:1
+                //
+                match imp.clone() {
+                    NImportResolved::Repin { .. } => {
+                        if let Some(existing_import) = imports.get(&key) {
+                            match existing_import {
+                                // replace
+                                p if matches!(
+                                    p,
+                                    NImportResolved::Pin { .. } | NImportResolved::Auto { .. }
+                                ) =>
+                                {
+                                    imports.insert(key, imp);
+                                }
+                                // do nothing (older repins have greater precedence)
+                                NImportResolved::Repin { .. } => {}
+                                // Should not be possible
+                                _ => {
+                                    return Err(anyhow::anyhow!(
+                                        "Internal error: cannot resolve requirement pins",
+                                    )
+                                    .into());
+                                }
+                            }
+                        } else {
+                            imports.insert(key, imp.clone());
+                        }
+                    }
+                    NImportResolved::Pin { pins: new_pins, .. } => {
+                        if let Some(existing_import) = imports.get_mut(&key) {
+                            match existing_import {
+                                // Check if pin is the same version, if same, do nothing, if not error
+                                NImportResolved::Pin { pins: existing_pins, .. } => {
+                                    existing_pins.extend(new_pins)
+                                }
+                                // do nothing
+                                NImportResolved::Repin { .. } => {}
+                                // Replace with new pin
+                                NImportResolved::Auto { .. } => {
+                                    imports.insert(key, imp);
+                                }
+                            }
+                        } else {
+                            imports.insert(key, imp.clone());
+                        }
+                    }
+                    NImportResolved::Auto { .. } => {
+                        if !imports.contains_key(&key) {
+                            imports.insert(key, imp);
+                        }
+                    }
                 }
             }
         }
-        imports.sort();
         Ok(imports)
     }
 }

backend/parsers/windmill-parser-py-imports/src/mapping.rs

@@ -142,6 +142,8 @@ pub static FULL_IMPORTS_MAP: PyMap = phf_map! {
     "google.cloud.dns" => "google-cloud-dns",
     "google.cloud.runtimeconfig" => "google-cloud-runtimeconfig",
     "google.cloud.iot" => "google-cloud-iot",
+    "google.generativeai" => "google-generativeai",
+    "google.genai" => "google-genai",
     // Azure
     "azure.mgmt.hybridkubernetes" => "azure-mgmt-hybridkubernetes",
     "azure.mgmt.sql" => "azure-mgmt-sql",
@@ -325,6 +327,7 @@ pub static FULL_IMPORTS_MAP: PyMap = phf_map! {
     "azure.mgmt.nspkg" => "azure-mgmt-nspkg",
     "azure.keyvault.secrets" => "azure-keyvault-secrets",
     "azure.storage.blob" => "azure-storage-blob",
+    "azure.storage.filedatalake" => "azure-storage-file-datalake",
     // Add new entry here ^
 };
 
@@ -376,5 +379,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
     "socks" => "PySocks",
     "taiga" => "python-taiga",
     "docx" => "python-docx",
+    "vt" => "vt-py",
     // Add new entry here ^
 };

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -19,7 +19,7 @@ def main():
 
 ";
         let mut already_visited = vec![];
-        let r = parse_python_imports(
+        let (r, ..) = parse_python_imports(
             code,
             "test-workspace",
             "f/foo/bar",
@@ -29,7 +29,15 @@ def main():
         )
         .await?;
         // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["matplotlib", "wmill", "zanzibar"]);
+        assert_eq!(
+            r,
+            vec![
+                "matplotlib # (mapped from matplotlib.pyplot)",
+                "wmill",
+                "zanzibar # (mapped from zanzibar.estonie)"
+            ]
+        );
+
         Ok(())
     }
 
@@ -52,7 +60,7 @@ def main():
 
 ";
         let mut already_visited = vec![];
-        let r = parse_python_imports(
+        let (r, ..) = parse_python_imports(
             code,
             "test-workspace",
             "f/foo/bar",
@@ -83,7 +91,7 @@ def main():
 ";
         let mut already_visited = vec![];
 
-        let r = parse_python_imports(
+        let (r, ..) = parse_python_imports(
             code,
             "test-workspace",
             "f/foo/bar",

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -49,8 +49,12 @@ impl Visit for ImportsFinder {
 pub fn parse_expr_for_imports(code: &str) -> anyhow::Result<Vec<String>> {
     let cm: Lrc<SourceMap> = Default::default();
     let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into());
+    let mut tss = TsSyntax::default();
+    tss.disallow_ambiguous_jsx_like;
+    tss.tsx = true;
+    tss.no_early_errors = true;
     let lexer = Lexer::new(
-        Syntax::Typescript(TsSyntax::default()),
+        Syntax::Typescript(tss),
         // EsVersion defaults to es5
         Default::default(),
         StringInput::from(&*fm),

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use anyhow::anyhow;
 use serde_json::json;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
@@ -208,15 +210,52 @@ pub struct AnsibleInventory {
     resource_type: Option<String>,
     pub pinned_resource: Option<String>,
 }
+
+#[derive(Debug, Clone)]
+pub struct GitRepo {
+    pub url: String,
+    pub commit: Option<String>,
+    pub branch: Option<String>,
+    pub target_path: String,
+}
+
 #[derive(Debug, Clone)]
 pub struct AnsibleRequirements {
     pub python_reqs: Vec<String>,
-    pub collections: Option<String>,
+    pub roles_and_collections: Option<String>,
     pub file_resources: Vec<FileResource>,
     pub inventories: Vec<AnsibleInventory>,
     pub vars: Vec<(String, String)>,
     pub resources: Vec<(String, String)>,
     pub options: AnsiblePlaybookOptions,
+    pub vault_password: Option<String>,
+    pub vault_id: Vec<String>,
+    pub git_repos: Vec<GitRepo>,
+    pub git_ssh_identity: Vec<String>,
+}
+
+impl Default for AnsibleRequirements {
+    fn default() -> Self {
+        Self {
+            python_reqs: vec![],
+            roles_and_collections: None,
+            file_resources: vec![],
+            inventories: vec![],
+            vars: vec![],
+            resources: vec![],
+            options: AnsiblePlaybookOptions {
+                verbosity: None,
+                forks: None,
+                timeout: None,
+                flush_cache: None,
+                force_handlers: None,
+            },
+            vault_password: None,
+            vault_id: vec![],
+            git_repos: vec![],
+            git_ssh_identity: vec![],
+        }
+    }
 }
 
 fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInventory>> {
@@ -275,22 +314,7 @@ pub fn parse_ansible_reqs(
         return Ok((logs, None, inner_content.to_string()));
     }
 
-    let opts = AnsiblePlaybookOptions {
-        verbosity: None,
-        forks: None,
-        timeout: None,
-        flush_cache: None,
-        force_handlers: None,
-    };
-    let mut ret = AnsibleRequirements {
-        python_reqs: vec![],
-        collections: None,
-        file_resources: vec![],
-        inventories: vec![],
-        vars: vec![],
-        resources: vec![],
-        options: opts,
-    };
+    let mut ret = AnsibleRequirements::default();
 
     if let Yaml::Hash(doc) = &docs[0] {
         for (key, value) in doc {
@@ -303,7 +327,7 @@ pub fn parse_ansible_reqs(
                             let mut out_str = String::new();
                             let mut emitter = YamlEmitter::new(&mut out_str);
                             emitter.dump(galaxy_requirements)?;
-                            ret.collections = Some(out_str);
+                            ret.roles_and_collections = Some(out_str);
                         }
                         if let Some(Yaml::Array(py_reqs)) =
                             deps.get(&Yaml::String("python".to_string()))
@@ -345,11 +369,60 @@ pub fn parse_ansible_reqs(
                 Yaml::String(key) if key == "inventory" => {
                     ret.inventories = parse_inventories(value)?;
                 }
+                Yaml::String(key) if key == "vault_password" => {
+                    let Yaml::String(filename) = value else {
+                        return Err(anyhow!(
+                            "Vault Password File expects a String containing the file name"
+                        ));
+                    };
+                    ret.vault_password = Some(filename.to_string());
+                }
+                Yaml::String(key) if key == "vault_id" => {
+                    let Yaml::Array(filenames) = value else {
+                        return Err(anyhow!("Vault ID field expects an array of strings in the format: `label@filename`"));
+                    };
+
+                    for f in filenames {
+                        let Yaml::String(filename) = f else {
+                            return Err(anyhow!("The elements of the vault_id field should be strings in the format: `label@filename`"));
+                        };
+                        ret.vault_id.push(filename.to_string());
+                    }
+                }
                 Yaml::String(key) if key == "options" => {
                     if let Yaml::Array(opts) = &value {
                         ret.options = parse_ansible_options(opts);
                     }
                 }
+                Yaml::String(key) if key == "git_repos" => {
+                    let Yaml::Array(repos) = &value else {
+                        return Err(anyhow!("git_repos field expects an array of repos"));
+                    };
+
+                    for r in repos {
+                        ret.git_repos.push(
+                            parse_git_repo(r)
+                                .map_err(|e| anyhow!("Failed to parse git repo: {e}"))?,
+                        );
+                    }
+                }
+                Yaml::String(key) if key == "git_ssh_identity" => {
+                    let Yaml::Array(indentities) = &value else {
+                        return Err(anyhow!(
+                            "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs"
+                        ));
+                    };
+
+                    for r in indentities {
+                        let Yaml::String(file_name) = r else {
+                            return Err(anyhow!(
+                                "Git ssh identity file must be a string path to a Windmill variable/secret"
+                            ));
+                        };
+
+                        ret.git_ssh_identity.push(file_name.clone());
+                    }
+                }
                 Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
                 _ => (),
             }
@@ -364,6 +437,38 @@ pub fn parse_ansible_reqs(
     Ok((logs, Some(ret), out_str))
 }
 
+fn parse_git_repo(r: &Yaml) -> anyhow::Result<GitRepo> {
+    let Yaml::Hash(repo) = r else {
+        return Err(anyhow!("Should be a Map"));
+    };
+
+    let url = repo
+        .get(&Yaml::String("url".to_string()))
+        .and_then(|v| v.as_str())
+        .map(|s| s.to_string())
+        .ok_or(anyhow!("Expected `url` field"))?;
+
+    let target_path = repo
+        .get(&Yaml::String("target".to_string()))
+        .and_then(|v| v.as_str())
+        .map(|s| s.to_string())
+        .ok_or(anyhow!(
+            "Expected `target` field (target directory for cloning the repo)"
+        ))?;
+
+    let branch = repo
+        .get(&Yaml::String("branch".to_string()))
+        .and_then(|v| v.as_str())
+        .map(|s| s.to_string());
+
+    let commit = repo
+        .get(&Yaml::String("commit".to_string()))
+        .and_then(|v| v.as_str())
+        .map(|s| s.to_string());
+
+    Ok(GitRepo { url, commit, branch, target_path })
+}
+
 fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
     let mut ret = AnsiblePlaybookOptions {
         verbosity: None,
@@ -529,3 +634,78 @@ fn yaml_to_json(yaml: &Yaml) -> serde_json::Value {
         _ => serde_json::Value::Null,
     }
 }
+
+fn update_versions(
+    section: &str,
+    yaml: &mut Yaml,
+    versions: &HashMap<String, String>,
+) -> anyhow::Result<String> {
+    let mut logs = String::new();
+
+    let Yaml::Hash(ref mut m) = yaml else {
+        return Err(anyhow!("{section} dependency should be a map"));
+    };
+
+    if let Some(Yaml::Array(elements)) = m.get_mut(&Yaml::String(section.to_string())) {
+        for el in elements {
+            let Yaml::Hash(ref mut h) = el else {
+                return Err(anyhow!("{section} dependency element should be a map"));
+            };
+
+            if let Some(name) = h
+                .get(&Yaml::String("name".to_string()))
+                .and_then(|n| n.as_str())
+            {
+                if let Some(version) = versions.get(name) {
+                    h.insert(
+                        Yaml::String("version".to_string()),
+                        Yaml::String(version.to_string()),
+                    );
+                } else {
+                    logs.push_str(&format!("WARNING: {section} dependency `{name}` has no locked version, using the latest or system installed version.\n"));
+                }
+            } else {
+                return Err(anyhow!(
+                    "{section} dependency element: missing or invalid `name` field"
+                ));
+            }
+        }
+    }
+
+    Ok(logs)
+}
+
+pub fn add_versions_to_requirements_yaml(
+    input: &str,
+    role_versions: &HashMap<String, String>,
+    collection_versions: &HashMap<String, String>,
+) -> anyhow::Result<(String,String)> {
+    let mut docs =
+        YamlLoader::load_from_str(input).map_err(|e| anyhow!("YAML parse error: {}", e))?;
+    let doc = &mut docs[0];
+
+    let mut logs = String::new();
+
+    logs.push_str(
+        &update_versions("roles", doc, role_versions)
+            .map_err(|e| anyhow!("Error updating role versions: {e}"))?,
+    );
+    logs.push_str(
+        &update_versions("collections", doc, collection_versions)
+            .map_err(|e| anyhow!("Error updating collection versions: {e}"))?,
+    );
+
+    if !logs.is_empty() {
+        logs.push_str("WARNING: You might want to try adding manual versions for these, otherwise there could be breaking changes on deployed scripts\n");
+    }
+
+    let mut out_str = String::new();
+    {
+        let mut emitter = YamlEmitter::new(&mut out_str);
+        emitter
+            .dump(doc)
+            .map_err(|e| anyhow!("YAML emit error: {}", e))?;
+    }
+
+    Ok((out_str, logs))
+}

backend/plot2.py

@@ -0,0 +1,33 @@
+import json
+import matplotlib.pyplot as plt
+
+# Path to the profiling JSON file
+# file_path = "/tmp/windmill/profiling_main.json"
+file_path = "/tmp/profiling.json"
+
+# Load the JSON data
+with open(file_path, "r") as f:
+    data = json.load(f)
+
+# Extract timings for "pre pull->post pull"
+pre_post_pull_timings = [
+    timing / 1000000.0 for entry in data["timings"]
+    for step, timing in entry["timings"]
+    # if step == "pre pull->post pull"
+    if step == "->job pulled from DB"
+]
+
+# Plotting the distribution
+plt.figure(figsize=(10, 6))
+# plt.hist(pre_post_pull_timings, bins=10, edgecolor='black')
+plt.scatter(range(len(pre_post_pull_timings)), pre_post_pull_timings,
+                alpha=1.0,          # Transparency level
+                s=40)                # Size of the dots`)
+plt.title("Distribution of 'pre pull->post pull' timings")
+# plt.xlabel("Time (ms)")
+# plt.ylabel("Frequency")
+plt.xlabel("Sample Index")
+plt.ylabel("Time (ms)")
+plt.grid(True)
+plt.tight_layout()
+plt.show()

backend/src/main.rs

@@ -8,11 +8,12 @@
 
 use anyhow::Context;
 use monitor::{
-    load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
+    load_base_url, load_otel, reload_critical_alerts_on_db_oversize,
+    reload_delete_logs_periodically_setting, reload_indexer_config,
     reload_instance_python_version_setting, reload_maven_repos_setting,
     reload_no_default_maven_setting, reload_nuget_config_setting,
     reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
-    send_logs_to_object_store,
+    send_logs_to_object_store, WORKERS_NAMES,
 };
 use rand::Rng;
 use sqlx::postgres::PgListener;
@@ -33,15 +34,16 @@ use windmill_common::{
     agent_workers::build_agent_http_client,
     get_database_url,
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
-        DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
-        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
-        NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING,
-        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
+        CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING,
+        ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
+        INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
+        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
         TIMEOUT_WAIT_RESULT_SETTING,
@@ -265,6 +267,8 @@ async fn windmill_main() -> anyhow::Result<()> {
 
     if mode == Mode::Standalone {
         println!("Running in standalone mode");
+    } else if mode == Mode::MCP {
+        println!("Running in MCP mode");
     }
 
     #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
@@ -297,7 +301,7 @@ async fn windmill_main() -> anyhow::Result<()> {
     }
 
     #[allow(unused_mut)]
-    let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
+    let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer || mode == Mode::MCP {
         0
     } else {
         std::env::var("NUM_WORKERS")
@@ -319,8 +323,9 @@ async fn windmill_main() -> anyhow::Result<()> {
         && (mode == Mode::Server || mode == Mode::Standalone);
 
     let indexer_mode = mode == Mode::Indexer;
+    let mcp_mode = mode == Mode::MCP;
 
-    let server_bind_address: IpAddr = if server_mode || indexer_mode {
+    let server_bind_address: IpAddr = if server_mode || indexer_mode || mcp_mode {
         std::env::var("SERVER_BIND_ADDR")
             .ok()
             .and_then(|x| x.parse().ok())
@@ -329,26 +334,16 @@ async fn windmill_main() -> anyhow::Result<()> {
         IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
     };
 
-    let mut first_worker_suffix = None;
-    let mut worker_names = vec![];
-
-    for _ in 0..num_workers {
+    let (conn, first_suffix) = if mode == Mode::Agent {
+        tracing::info!(
+            "Creating http client for cluster using base internal url {}",
+            std::env::var("BASE_INTERNAL_URL").unwrap_or_default()
+        );
         let suffix = windmill_common::utils::worker_suffix(&hostname, &rd_string(5));
-        worker_names.push(windmill_common::utils::worker_name_with_suffix(
-            mode == Mode::Agent,
-            WORKER_GROUP.as_str(),
-            &suffix,
-        ));
-        if first_worker_suffix.is_none() {
-            first_worker_suffix = Some(suffix);
-        }
-    }
-
-    let conn = if mode == Mode::Agent {
-        let worker_suffix = first_worker_suffix.unwrap_or_else(|| {
-            panic!("there must be at least one worker in agent mode");
-        });
-        Connection::Http(build_agent_http_client(&worker_suffix))
+        (
+            Connection::Http(build_agent_http_client(&suffix)),
+            Some(suffix),
+        )
     } else {
         println!("Connecting to database...");
 
@@ -366,7 +361,7 @@ async fn windmill_main() -> anyhow::Result<()> {
         load_otel(&db).await;
 
         tracing::info!("Database connected");
-        Connection::Sql(db)
+        (Connection::Sql(db), None)
     };
 
     let environment = load_base_url(&conn)
@@ -390,7 +385,7 @@ async fn windmill_main() -> anyhow::Result<()> {
         .is_some_and(|x| x == "1" || x == "true");
 
     if let Some(db) = conn.as_sql() {
-        if !is_agent && !indexer_mode {
+        if !is_agent && !indexer_mode && !mcp_mode {
             let skip_migration = std::env::var("SKIP_MIGRATION")
                 .map(|val| val == "true")
                 .unwrap_or(false);
@@ -409,6 +404,7 @@ async fn windmill_main() -> anyhow::Result<()> {
     let conn = if mode == Mode::Agent {
         conn
     } else {
+        // This time we use a pool of connections
         let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?;
         Connection::Sql(db)
     };
@@ -439,16 +435,6 @@ Windmill Community Edition {GIT_VERSION}
 
     display_config(&ENV_SETTINGS);
 
-    if let Err(e) = reload_base_url_setting(&conn).await {
-        tracing::error!("Error loading base url: {:?}", e)
-    }
-
-    if let Some(db) = conn.as_sql() {
-        if let Err(e) = reload_critical_error_channels_setting(&db).await {
-            tracing::error!("Could loading critical error emails setting: {:?}", e);
-        }
-    }
-
     #[cfg(feature = "enterprise")]
     {
         // load the license key and check if it's valid
@@ -462,7 +448,7 @@ Windmill Community Edition {GIT_VERSION}
         if !valid_key && !server_mode {
             tracing::error!("Invalid license key, workers require a valid license key");
         }
-        if server_mode {
+        if server_mode || mcp_mode {
             if let Some(db) = conn.as_sql() {
                 // only force renewal if invalid but not empty (= expired)
                 let renewed_now = maybe_renew_license_key_on_start(
@@ -482,10 +468,10 @@ Windmill Community Edition {GIT_VERSION}
         }
     }
 
-    if server_mode || worker_mode || indexer_mode {
+    if server_mode || worker_mode || indexer_mode || mcp_mode {
         let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
 
-        let port = if server_mode || indexer_mode {
+        let port = if server_mode || indexer_mode || mcp_mode {
             port_var.unwrap_or(DEFAULT_PORT as u16)
         } else {
             port_var.unwrap_or(0)
@@ -666,6 +652,7 @@ Windmill Community Edition {GIT_VERSION}
                         server_killpill_rx,
                         base_internal_tx,
                         server_mode,
+                        mcp_mode,
                         base_internal_url.clone(),
                     )
                     .await?;
@@ -686,14 +673,34 @@ Windmill Community Edition {GIT_VERSION}
             if !killpill_rx.try_recv().is_ok() {
                 let base_internal_url = base_internal_rx.await?;
                 if worker_mode {
+                    let mut workers = vec![];
+                    for i in 0..num_workers {
+                        let suffix: String = if i == 0 && first_suffix.as_ref().is_some() {
+                            first_suffix.as_ref().unwrap().clone()
+                        } else {
+                            windmill_common::utils::worker_suffix(&hostname, &rd_string(5))
+                        };
+                        let worker_conn = WorkerConn {
+                            conn: if i == 0 || mode != Mode::Agent {
+                                conn.clone()
+                            } else {
+                                Connection::Http(build_agent_http_client(&suffix))
+                            },
+                            worker_name: windmill_common::utils::worker_name_with_suffix(
+                                mode == Mode::Agent,
+                                WORKER_GROUP.as_str(),
+                                &suffix,
+                            ),
+                        };
+                        workers.push(worker_conn);
+                    }
+
                     run_workers(
-                        conn.clone(),
                         rx,
                         killpill_tx.clone(),
-                        num_workers,
                         base_internal_url.clone(),
                         hostname.clone(),
-                        &worker_names,
+                        &workers,
                     )
                     .await?;
                     tracing::info!("All workers exited.");
@@ -912,6 +919,12 @@ Windmill Community Edition {GIT_VERSION}
                                                                 tracing::error!(error = %e, "Could not reload critical error emails setting");
                                                             }
                                                         },
+                                                        CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => {
+                                                            if let Err(e) = reload_critical_alerts_on_db_oversize(&db).await {
+                                                                tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting");
+                                                            }
+
+                                                        },
                                                         JWT_SECRET_SETTING => {
                                                             if let Err(e) = reload_jwt_secret_setting(&db).await {
                                                                 tracing::error!(error = %e, "Could not reload jwt secret setting");
@@ -1048,15 +1061,19 @@ Windmill Community Edition {GIT_VERSION}
             }
         }
 
-        futures::try_join!(
-            shutdown_signal,
-            workers_f,
-            monitor_f,
-            server_f,
-            metrics_f,
-            indexer_f,
-            log_indexer_f
-        )?;
+        if mcp_mode {
+            futures::try_join!(shutdown_signal, workers_f, server_f)?;
+        } else {
+            futures::try_join!(
+                shutdown_signal,
+                workers_f,
+                monitor_f,
+                server_f,
+                metrics_f,
+                indexer_f,
+                log_indexer_f
+            )?;
+        }
     } else {
         tracing::info!("Nothing to do, exiting.");
     }
@@ -1134,16 +1151,20 @@ fn display_config(envs: &[&str]) {
     )
 }
 
+pub struct WorkerConn {
+    conn: Connection,
+    worker_name: String,
+}
+
 pub async fn run_workers(
-    db: Connection,
     mut rx: tokio::sync::broadcast::Receiver<()>,
     tx: KillpillSender,
-    num_workers: i32,
     base_internal_url: String,
     hostname: String,
-    worker_names: &[String],
+    workers: &[WorkerConn],
 ) -> anyhow::Result<()> {
     let mut killpill_rxs = vec![];
+    let num_workers = workers.len();
     for _ in 0..num_workers {
         killpill_rxs.push(rx.resubscribe());
     }
@@ -1201,9 +1222,12 @@ pub async fn run_workers(
         "Starting {num_workers} workers and SLEEP_QUEUE={}ms",
         *windmill_worker::SLEEP_QUEUE
     );
+
     for i in 1..(num_workers + 1) {
-        let db1 = db.clone();
-        let worker_name = worker_names[i as usize - 1].clone();
+        let wk_conf = &workers[i as usize - 1];
+        let conn1 = wk_conf.conn.clone();
+        let worker_name = wk_conf.worker_name.clone();
+        WORKERS_NAMES.write().await.push(worker_name.clone());
         let ip = ip.clone();
         let rx = killpill_rxs.pop().unwrap();
         let tx = tx.clone();
@@ -1216,7 +1240,7 @@ pub async fn run_workers(
             }
 
             let f = windmill_worker::run_worker(
-                &db1,
+                &conn1,
                 &hostname,
                 worker_name,
                 i as u64,

backend/src/monitor.rs

@@ -13,7 +13,7 @@ use std::{
 
 use chrono::{NaiveDateTime, Utc};
 use futures::{stream::FuturesUnordered, StreamExt};
-use serde::de::DeserializeOwned;
+use serde::{de::DeserializeOwned, Deserialize};
 use sqlx::{Pool, Postgres};
 use tokio::{
     join,
@@ -30,28 +30,54 @@ use windmill_api::{
 
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
+#[cfg(feature = "enterprise")]
+use windmill_common::ee::low_disk_alerts;
 
 #[cfg(feature = "oauth2")]
 use windmill_common::global_settings::OAUTH_SETTING;
 use windmill_common::{
-    utils::empty_string_as_none,
-    agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
-        DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+    agent_workers::DECODED_AGENT_TOKEN,
+    auth::create_token_for_owner,
+    ee::CriticalErrorChannel,
+    error,
+    flow_status::{FlowStatus, FlowStatusModule},
+    global_settings::{
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
+        CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
+        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
+        HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
+        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
+        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING,
+        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
-    }, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{
-        load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP
-    }, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
+    },
+    indexer::load_indexer_config,
+    jobs::QueuedJob,
+    jwt::JWT_SECRET,
+    oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
+    server::load_smtp_config,
+    tracing_init::JSON_FMT,
+    users::truncate_token,
+    utils::empty_string_as_none,
+    utils::{now_from_db, rd_string, report_critical_error, Mode},
+    worker::{
+        load_worker_config, reload_custom_tags_setting, store_pull_query,
+        store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE,
+        DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR,
+        WORKER_CONFIG, WORKER_GROUP,
+    },
+    KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED,
+    CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
+    METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED,
+    OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
 };
 use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
 use windmill_worker::{
-    handle_job_error, AuthedClient, JobCompletedSender,  SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL
+    handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES,
+    INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN,
+    NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
 };
 
 #[cfg(feature = "parquet")]
@@ -111,7 +137,7 @@ lazy_static::lazy_static! {
     .and_then(|x| x.parse::<bool>().ok())
     .unwrap_or(false);
 
-
+    pub static ref WORKERS_NAMES: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
 
     static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
@@ -135,7 +161,6 @@ pub async fn initial_load(
         }
     }
 
-    
     if let Err(e) = load_metrics_enabled(conn).await {
         tracing::error!("Error loading expose metrics: {e:#}");
     }
@@ -161,6 +186,12 @@ pub async fn initial_load(
     if server_mode {
         if let Some(db) = conn.as_sql() {
             load_require_preexisting_user(db).await;
+            if let Err(e) = reload_critical_alerts_on_db_oversize(db).await {
+                tracing::error!(
+                    "Error reloading critical alerts on db oversize setting: {:?}",
+                    e
+                )
+            }
         }
     }
 
@@ -172,11 +203,13 @@ pub async fn initial_load(
             }
             Connection::Http(_) => {
                 // TODO: reload worker config from http
-                WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN.as_ref().map(|x| x.tags.clone()).unwrap_or_default();
+                WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN
+                    .as_ref()
+                    .map(|x| x.tags.clone())
+                    .unwrap_or_default();
             }
         }
     }
-    
 
     if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
         tracing::error!("Error reloading hub base url: {:?}", e)
@@ -190,7 +223,6 @@ pub async fn initial_load(
         if let Err(e) = reload_custom_tags_setting(db).await {
             tracing::error!("Error reloading custom tags: {:?}", e)
         }
-
     }
 
     #[cfg(feature = "parquet")]
@@ -225,7 +257,8 @@ pub async fn initial_load(
 }
 
 pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> {
-    let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await;
+    let metrics_enabled =
+        load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await;
     match metrics_enabled {
         Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed),
         _ => (),
@@ -345,13 +378,13 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::
         load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await
     {
         CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
-
     }
     Ok(())
 }
 
 pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> {
-    let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await;
+    let metrics_enabled =
+        load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await;
     match metrics_enabled {
         Ok(Some(serde_json::Value::Bool(t))) => {
             METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed);
@@ -575,7 +608,9 @@ async fn send_log_file_to_object_store(
         };
 
         let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| {
-            last_log_file_sent.map(|last_log_file_sent| last_log_file_sent >= ts).unwrap_or(false)
+            last_log_file_sent
+                .map(|last_log_file_sent| last_log_file_sent >= ts)
+                .unwrap_or(false)
         });
 
         if exists.unwrap_or(false) {
@@ -612,7 +647,9 @@ async fn send_log_file_to_object_store(
         let (ok_lines, err_lines) = read_log_counters(ts_str);
 
         if let Some(db) = conn.as_sql() {
-            if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)", 
+            if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)
+             VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)
+             ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7",
                 hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
                 .execute(db)
                 .await {
@@ -1002,11 +1039,21 @@ pub async fn reload_nuget_config_setting(conn: &Connection) {
     .await;
 }
 pub async fn reload_maven_repos_setting(conn: &Connection) {
-    reload_option_setting_with_tracing(conn, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
-        .await;
+    reload_option_setting_with_tracing(
+        conn,
+        windmill_common::global_settings::MAVEN_REPOS_SETTING,
+        "MAVEN_REPOS",
+        MAVEN_REPOS.clone(),
+    )
+    .await;
 }
 pub async fn reload_no_default_maven_setting(conn: &Connection) {
-    let value = load_value_from_global_settings_with_conn(conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true).await;
+    let value = load_value_from_global_settings_with_conn(
+        conn,
+        windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING,
+        true,
+    )
+    .await;
     match value {
         Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed),
         Err(e) => {
@@ -1175,7 +1222,6 @@ pub async fn load_value_from_global_settings(
     Ok(r)
 }
 
-
 pub async fn load_value_from_global_settings_with_conn(
     conn: &Connection,
     setting_name: &str,
@@ -1185,14 +1231,18 @@ pub async fn load_value_from_global_settings_with_conn(
         Connection::Sql(db) => Ok(load_value_from_global_settings(db, setting_name).await?),
         Connection::Http(client) => {
             if load_from_http {
-                client.get::<Option<serde_json::Value>>(&format!("/api/agent_workers/get_global_setting/{}", setting_name)).await
-                .map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e))
+                client
+                    .get::<Option<serde_json::Value>>(&format!(
+                        "/api/agent_workers/get_global_setting/{}",
+                        setting_name
+                    ))
+                    .await
+                    .map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e))
             } else {
                 Ok(None)
             }
         }
     }
-
 }
 
 pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
@@ -1314,12 +1364,12 @@ pub async fn monitor_db(
     let zombie_jobs_f = async {
         if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING {
             if let Some(db) = conn.as_sql() {
-            handle_zombie_jobs(db, base_internal_url, "server").await;
-            match handle_zombie_flows(db).await {
-                Err(err) => {
-                    tracing::error!("Error handling zombie flows: {:?}", err);
-                },
-                _ => {}
+                handle_zombie_jobs(db, base_internal_url, "server").await;
+                match handle_zombie_flows(db).await {
+                    Err(err) => {
+                        tracing::error!("Error handling zombie flows: {:?}", err);
+                    }
+                    _ => {}
                 }
             }
         }
@@ -1327,7 +1377,7 @@ pub async fn monitor_db(
     let expired_items_f = async {
         if server_mode && !initial_load {
             if let Some(db) = conn.as_sql() {
-            delete_expired_items(&db).await;
+                delete_expired_items(&db).await;
             }
         }
     };
@@ -1365,6 +1415,23 @@ pub async fn monitor_db(
         }
     };
 
+    let low_disk_alerts_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Some(db) = conn.as_sql() {
+            low_disk_alerts(
+                &db,
+                server_mode,
+                _worker_mode,
+                WORKERS_NAMES.read().await.clone(),
+            )
+            .await;
+        }
+        #[cfg(not(feature = "enterprise"))]
+        {
+            ()
+        }
+    };
+
     let apply_autoscaling_f = async {
         #[cfg(feature = "enterprise")]
         if server_mode && !initial_load {
@@ -1387,6 +1454,7 @@ pub async fn monitor_db(
         verify_license_key_f,
         worker_groups_alerts_f,
         jobs_waiting_alerts_f,
+        low_disk_alerts_f,
         apply_autoscaling_f,
         update_min_worker_version_f,
     );
@@ -1496,11 +1564,7 @@ pub async fn reload_indexer_config(db: &Pool<Postgres>) {
     }
 }
 
-pub async fn reload_worker_config(
-    db: &DB,
-    tx: KillpillSender,
-    kill_if_change: bool,
-) {
+pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: bool) {
     let config = load_worker_config(db, tx.clone()).await;
     if let Err(e) = config {
         tracing::error!("Error reloading worker config: {:?}", e)
@@ -1543,7 +1607,8 @@ pub async fn reload_worker_config(
 }
 
 pub async fn load_base_url(conn: &Connection) -> error::Result<String> {
-    let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?;
+    let q_base_url =
+        load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?;
 
     let std_base_url = std::env::var("BASE_URL")
         .ok()
@@ -1574,10 +1639,9 @@ pub async fn load_base_url(conn: &Connection) -> error::Result<String> {
 }
 
 pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
-
     #[cfg(feature = "oauth2")]
     let oauths = if let Some(db) = conn.as_sql() {
-        let q_oauth = load_value_from_global_settings (db, OAUTH_SETTING).await?;
+        let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
 
         if let Some(q) = q_oauth {
             if let Ok(v) = serde_json::from_value::<
@@ -1652,7 +1716,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
             increment_counter AS (
                 INSERT INTO zombie_job_counter (job_id, counter)
                 SELECT id, 1 FROM to_update WHERE counter < $2
-                ON CONFLICT (job_id) DO UPDATE 
+                ON CONFLICT (job_id) DO UPDATE
                 SET counter = zombie_job_counter.counter + 1
             ),
             update_concurrency AS (
@@ -1744,7 +1808,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
 
     let same_worker_timeout_jobs = {
         let long_same_worker_jobs = sqlx::query!(
-            "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval 
+            "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval
     AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker",
         )
         .fetch_all(db)
@@ -1758,9 +1822,9 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
             .collect::<Vec<_>>();
 
         let long_dead_workers: std::collections::HashSet<String> = sqlx::query_scalar!(
-            "WITH worker_ids AS (SELECT unnest($1::text[]) as worker) 
-            SELECT worker_ids.worker FROM worker_ids 
-            LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker 
+            "WITH worker_ids AS (SELECT unnest($1::text[]) as worker)
+            SELECT worker_ids.worker FROM worker_ids
+            LEFT JOIN worker_ping ON worker_ids.worker = worker_ping.worker
                 WHERE worker_ping.worker IS NULL OR worker_ping.ping_at < now() - ('60 seconds')::interval",
             &worker_ids[..]
         )
@@ -1804,7 +1868,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
     let non_restartable_jobs = if *RESTART_ZOMBIE_JOBS {
         vec![]
     } else {
-        sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval 
+        sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE last_ping < now() - ($1 || ' seconds')::interval
     AND running = true  AND job_kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow') AND same_worker = false")
         .bind(ZOMBIE_JOB_TIMEOUT.as_str())
         .fetch_all(db)
@@ -1863,7 +1927,8 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
             mpsc::channel::<SameWorkerPayload>(1);
         let same_worker_tx_never_used =
             SameWorkerSender(same_worker_tx_never_used, Arc::new(AtomicU16::new(0)));
-        let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used();
+        let (send_result_never_used, _send_result_rx_never_used) =
+            JobCompletedSender::new_never_used();
 
         let label = if job.permissioned_as != format!("u/{}", job.created_by)
             && job.permissioned_as != job.created_by
@@ -1914,7 +1979,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
             worker_name,
             send_result_never_used,
             #[cfg(feature = "benchmark")]
-            &mut windmill_worker::bench::BenchmarkIter::new(),
+            &mut windmill_common::bench::BenchmarkIter::new(),
         )
         .await;
     }
@@ -2001,7 +2066,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
                 }
             );
             report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
-            cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, 
+            cancel_zombie_flow_job(db, flow.id, &flow.workspace_id,
                 format!(r#"{reason}
 This would happen if a worker was interrupted, killed or crashed while doing a state transition at the end of a job which is always an unexpected behavior that should never happen.
 Please check your worker logs for more details and feel free to report it to the Windmill team on our Discord or support@windmill.dev (response for non EE customers will be best effort) with as much context as possible, ideally:
@@ -2018,7 +2083,7 @@ Please check your worker logs for more details and feel free to report it to the
         r#"
         DELETE
         FROM parallel_monitor_lock
-        WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval 
+        WHERE last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval
         RETURNING parent_flow_id, job_id, last_ping, (SELECT workspace_id FROM v2_job_queue q
             WHERE q.id = parent_flow_id AND q.running = true AND q.canceled_by IS NULL
         ) AS workspace_id
@@ -2073,8 +2138,12 @@ async fn cancel_zombie_flow_job(
     Ok(())
 }
 
-pub async fn reload_hub_base_url_setting(conn: &Connection, server_mode: bool) -> error::Result<()> {
-    let hub_base_url = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?;
+pub async fn reload_hub_base_url_setting(
+    conn: &Connection,
+    server_mode: bool,
+) -> error::Result<()> {
+    let hub_base_url =
+        load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?;
 
     let base_url = if let Some(q) = hub_base_url {
         if let Ok(v) = serde_json::from_value::<String>(q.clone()) {
@@ -2126,7 +2195,7 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<
             v
         } else {
             tracing::error!(
-                "Could not parse critical_error_emails setting as an array of channels, found: {:#?}",
+                "Could not parse critical_error_channels setting as an array of channels, found: {:#?}",
                 &q
             );
             vec![]
@@ -2141,6 +2210,37 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<
     Ok(())
 }
 
+pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<()> {
+    #[derive(Deserialize)]
+    struct DBOversize {
+        enabled: bool,
+        value: f32,
+    }
+    let db_oversize_value =
+        load_value_from_global_settings(conn, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING).await?;
+
+    let db_oversize = if let Some(q) = db_oversize_value {
+        match serde_json::from_value::<DBOversize>(q.clone()) {
+            Ok(DBOversize { enabled: true, value }) => Some(value),
+            Ok(_) => None,
+            Err(q) => {
+                tracing::error!(
+                    "Could not parse critical_alerts_on_db_oversize setting, found: {:#?}",
+                    &q
+                );
+                None
+            }
+        }
+    } else {
+        None
+    };
+
+    let mut l = CRITICAL_ALERTS_ON_DB_OVERSIZE.write().await;
+    *l = db_oversize;
+
+    Ok(())
+}
+
 async fn generate_and_save_jwt_secret(db: &DB) -> error::Result<String> {
     let secret = rd_string(32);
     sqlx::query!(

backend/tests/fixtures/lockfile_python.sql

@@ -0,0 +1,51 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+# requirements:
+# microdot==2.2.0 
+
+import pandas
+import requests
+import tiny # pin: tiny==0.1.2
+
+def main():
+    pass
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/requirements', 12346, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+# extra_requirements:
+# bottle==0.13.2
+
+import tiny
+
+def main():
+    pass
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/extra_requirements', 12347, 'python3', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import tiny # pin: bottle==0.13.2
+import simplejson # pin: simplejson==3.19.3 
+
+def main():
+    return [test1(), test2(), test3(), test4()]
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/pins', 12348, 'python3', '');

backend/tests/worker.rs

@@ -16,16 +16,20 @@ use tokio::sync::RwLock;
 #[cfg(feature = "enterprise")]
 use tokio::time::{timeout, Duration};
 
+#[cfg(feature = "python")]
 use windmill_api_client::types::{CreateFlowBody, RawScript};
 #[cfg(feature = "enterprise")]
 use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
 use windmill_api_client::types::{NewScript, ScriptLang as NewScriptLanguage};
 
 use serde::Serialize;
+#[cfg(feature = "deno_core")]
+use windmill_common::flows::InputTransform;
 use windmill_common::worker::WORKER_CONFIG;
+
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule, RestartedFrom},
-    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
+    flows::{FlowModule, FlowModuleValue, FlowValue},
     jobs::{JobKind, JobPayload, RawCode},
     jwt::JWT_SECRET,
     scripts::{ScriptHash, ScriptLang},
@@ -137,10 +141,11 @@ impl ApiServer {
             rx,
             port_tx,
             false,
+            false,
             format!("http://localhost:{}", addr.port()),
         ));
 
-        _port_rx.await.unwrap();
+        _port_rx.await.expect("failed to receive port");
 
         // clear the cache between tests
         windmill_common::cache::clear();
@@ -166,6 +171,7 @@ impl ApiServer {
 //     Ok(())
 // }
 
+#[cfg(feature = "python")]
 fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     cjob.flow_status.clone().and_then(|fs| {
         find_module_in_vec(
@@ -175,6 +181,7 @@ fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     })
 }
 
+#[cfg(feature = "python")]
 fn find_module_in_vec(modules: Vec<FlowStatusModule>, id: &str) -> Option<FlowStatusModule> {
     modules.into_iter().find(|s| s.id() == id)
 }
@@ -283,6 +290,7 @@ mod suspend_resume {
             .unwrap()
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn test(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -365,6 +373,7 @@ mod suspend_resume {
         );
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn cancel_from_job(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -390,6 +399,7 @@ mod suspend_resume {
         );
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn cancel_after_suspend(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -563,6 +573,7 @@ def main(last, port):
         .unwrap()
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn test_pass(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -608,6 +619,7 @@ def main(last, port):
         assert_eq!(json!([3, 5, 7, 9]), result);
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn test_fail_step_zero(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -651,6 +663,7 @@ def main(last, port):
         );
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base"))]
     async fn test_fail_step_one(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -692,6 +705,7 @@ def main(last, port):
             .contains("index out of range"));
     }
 
+    #[cfg(feature = "python")]
     #[sqlx::test(fixtures("base"))]
     async fn test_with_failure_module(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -768,6 +782,7 @@ def main(error, port):
     }
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_iteration(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -826,6 +841,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .contains("2"));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_iteration_parallel(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1104,6 +1120,7 @@ trait StreamFind: futures::Stream + Unpin + Sized {
 
 impl<T: futures::Stream + Unpin + Sized> StreamFind for T {}
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_deno_flow(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1222,6 +1239,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
     }
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_identity(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1259,6 +1277,7 @@ async fn test_identity(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!(42));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1534,6 +1553,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([[42]]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_stop_after_if(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1587,6 +1607,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
     assert_eq!(json!(-123), result);
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_stop_after_if_nested(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1645,6 +1666,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
     assert_eq!(json!([-123]), result);
 }
 
+#[cfg(all(feature = "deno_core", feature = "python"))]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_flow(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1702,6 +1724,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
     }
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_flow_2(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -1776,6 +1799,7 @@ func main(derp string) (string, error) {
     assert_eq!(result, serde_json::json!("hello world"));
 }
 
+#[cfg(feature = "rust")]
 #[sqlx::test(fixtures("base"))]
 async fn test_rust_job(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2031,6 +2055,7 @@ public class Main {
     assert_eq!(job.json_result(), Some(json!("hello world")));
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_job(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2064,6 +2089,7 @@ def main():
     assert_eq!(result, serde_json::json!("hello world"));
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_job_heavy_dep(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2100,6 +2126,7 @@ def main():
     assert_eq!(result, serde_json::json!(3));
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_job_with_imports(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2203,6 +2230,7 @@ export async function main(a: Date) {
     assert_eq!(result, serde_json::json!("object"));
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2238,6 +2266,7 @@ def main(a: datetime, b: bytes):
     assert_eq!(result, serde_json::json!([true, true]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_empty_loop_1(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2294,6 +2323,7 @@ async fn test_empty_loop_1(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!(0));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_invalid_first_step(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2374,6 +2404,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_step_after_loop(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2497,6 +2528,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([1, 2]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_branchone_with_cond(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2533,6 +2565,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([1, 3]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_branchall_sequential(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2571,6 +2604,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_branchall_simple(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2698,6 +2732,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     );
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_branchone_nested(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2817,6 +2852,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
     );
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 async fn test_failure_module(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -2929,6 +2965,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_flow_lock_all(db: Pool<Postgres>) {
     use futures::StreamExt;
@@ -3067,6 +3104,7 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
         });
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base"))]
 
 async fn test_complex_flow_restart(db: Pool<Postgres>) {
@@ -3751,6 +3789,7 @@ export async function main() {
     run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
 }
 
+#[cfg(feature = "deno_core")]
 #[sqlx::test(fixtures("base", "relative_bun"))]
 async fn test_nested_imports_bun(db: Pool<Postgres>) {
     let content = r#"
@@ -3799,6 +3838,7 @@ export async function main() {
     run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base", "relative_python"))]
 async fn test_relative_imports_python(db: Pool<Postgres>) {
     let content = r#"
@@ -3816,6 +3856,7 @@ def main():
     run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
 }
 
+#[cfg(feature = "python")]
 #[sqlx::test(fixtures("base", "relative_python"))]
 async fn test_nested_imports_python(db: Pool<Postgres>) {
     let content = r#"
@@ -3831,6 +3872,212 @@ def main():
     run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
 }
 
+#[cfg(feature = "python")]
+async fn assert_lockfile(
+    db: &Pool<Postgres>,
+    script_content: String,
+    language: ScriptLang,
+    expected_lines: Vec<&str>,
+) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    client
+        .create_script(
+            "test-workspace",
+            &NewScript {
+                language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
+                content: script_content,
+                path: "f/system/test_import".to_string(),
+                concurrent_limit: None,
+                concurrency_time_window_s: None,
+                cache_ttl: None,
+                dedicated_worker: None,
+                description: "".to_string(),
+                draft_only: None,
+                envs: vec![],
+                is_template: None,
+                kind: None,
+                parent_hash: None,
+                lock: None,
+                summary: "".to_string(),
+                tag: None,
+                schema: std::collections::HashMap::new(),
+                ws_error_handler_muted: Some(false),
+                priority: None,
+                delete_after_use: None,
+                timeout: None,
+                restart_unless_cancelled: None,
+                deployment_message: None,
+                concurrency_key: None,
+                visible_to_runner_only: None,
+                no_main_func: None,
+                codebase: None,
+                has_preprocessor: None,
+                on_behalf_of_email: None,
+            },
+        )
+        .await
+        .unwrap();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            completed.next().await; // deployed script
+
+            let script = sqlx::query!(
+                "SELECT hash FROM script WHERE path = $1",
+                "f/system/test_import".to_string()
+            )
+            .fetch_one(&db2)
+            .await
+            .unwrap();
+
+            let job = RunJob::from(JobPayload::Dependencies {
+                path: "f/system/test_import".to_string(),
+                hash: ScriptHash(script.hash),
+                dedicated_worker: None,
+                language,
+            })
+            .push(&db2)
+            .await;
+
+            completed.next().await; // completed job
+
+            let result = completed_job(job, &db2).await.json_result().unwrap();
+
+            assert_eq!(
+                result,
+                json!({
+                    "lock": expected_lines.join("\n"),
+                    "status": "Successful lock file generation"
+                })
+            );
+        },
+        port,
+    )
+    .await;
+}
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "lockfile_python"))]
+async fn test_requirements_python(db: Pool<Postgres>) {
+    let content = r#"
+# py311
+# requirements:
+# tiny==0.1.3
+
+import bar
+import baz # pin: foo
+import baz # repin: fee
+import bug # repin: free
+    
+def main():
+    pass
+"#
+    .to_string();
+
+    assert_lockfile(
+        &db,
+        content,
+        ScriptLang::Python3,
+        vec!["# py311", "tiny==0.1.3"],
+    )
+    .await;
+}
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "lockfile_python"))]
+async fn test_extra_requirements_python(db: Pool<Postgres>) {
+    {
+        let content = r#"
+# py311
+# extra_requirements:
+# tiny
+
+import f.system.extra_requirements
+import tiny # pin: tiny==0.1.0
+import tiny # pin: tiny==0.1.1
+import tiny # repin: tiny==0.1.2
+
+def main():
+    pass
+    "#
+        .to_string();
+
+        assert_lockfile(
+            &db,
+            content,
+            ScriptLang::Python3,
+            vec!["# py311", "bottle==0.13.2", "tiny==0.1.2"],
+        )
+        .await;
+    }
+}
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "lockfile_python"))]
+async fn test_extra_requirements_python2(db: Pool<Postgres>) {
+    let content = r#"
+# py311
+# extra_requirements:
+# tiny==0.1.3
+
+import simplejson # pin: simplejson==3.20.1
+def main():
+    pass
+"#
+    .to_string();
+
+    assert_lockfile(
+        &db,
+        content,
+        ScriptLang::Python3,
+        vec!["# py311", "simplejson==3.20.1", "tiny==0.1.3"],
+    )
+    .await;
+}
+
+#[cfg(feature = "python")]
+#[sqlx::test(fixtures("base", "lockfile_python"))]
+async fn test_pins_python(db: Pool<Postgres>) {
+    let content = r#"
+# py311
+# extra_requirements:
+# tiny==0.1.3
+# bottle==0.13.2
+
+import f.system.requirements
+import f.system.pins
+import tiny # repin: tiny==0.1.3
+import simplejson
+
+def main():
+    pass
+"#
+    .to_string();
+
+    assert_lockfile(
+        &db,
+        content,
+        ScriptLang::Python3,
+        vec![
+            "# py311",
+            "bottle==0.13.2",
+            "microdot==2.2.0",
+            "simplejson==3.19.3",
+            "tiny==0.1.3",
+        ],
+    )
+    .await;
+}
 #[sqlx::test(fixtures("base", "result_format"))]
 async fn test_result_format(db: Pool<Postgres>) {
     let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41";
@@ -4082,6 +4329,7 @@ mod job_payload {
         ];
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_script_hash_payload(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4242,6 +4490,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_flow_node_payload(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4426,6 +4675,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_flow_payload(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4468,6 +4718,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_flow_payload_with_preprocessor(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4535,6 +4786,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_restarted_flow_payload(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4587,6 +4839,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_raw_flow_payload(db: Pool<Postgres>) {
         initialize_tracing().await;
@@ -4633,6 +4886,7 @@ mod job_payload {
         test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
     }
 
+    #[cfg(feature = "deno_core")]
     #[sqlx::test(fixtures("base", "hello"))]
     async fn test_raw_flow_payload_with_restarted_from(db: Pool<Postgres>) {
         initialize_tracing().await;

backend/windmill-api/Cargo.toml

@@ -13,9 +13,9 @@ default = []
 enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
 stripe = []
 agent_worker_server = []
-enterprise_saml = ["dep:samael"]
+enterprise_saml = ["dep:samael", "dep:libxml"]
 benchmark = []
-embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"]
+embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
 parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
 openidconnect = ["dep:openidconnect"]
@@ -32,10 +32,13 @@ static_frontend = ["dep:rust-embed"]
 postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"]
 mqtt_trigger = ["dep:thiserror", "dep:rumqttc"]
 sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"]
+deno_core = ["dep:deno_core", "dep:deno_error"]
 gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
 cloud = ["windmill-common/cloud"]
+mcp = ["dep:rmcp"]
 
 [dependencies]
+rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true }
 windmill-queue.workspace = true
 windmill-common = { workspace = true, default-features = false }
 windmill-audit.workspace = true
@@ -47,10 +50,10 @@ windmill-git-sync.workspace = true
 windmill-indexer = { workspace = true, optional = true }
 windmill-worker.workspace = true
 tokio.workspace = true
+tokio-stream.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
 axum.workspace = true
-half = { workspace = true, optional = true}
 futures.workspace = true
 git-version.workspace = true
 tower.workspace = true
@@ -98,6 +101,7 @@ async_zip = { workspace = true, optional = true }
 regex.workspace = true
 bytes.workspace = true
 samael = { workspace = true, optional = true }
+libxml = { workspace = true, optional = true }
 async-recursion.workspace = true
 rsa = { workspace = true, optional = true}
 uuid.workspace = true
@@ -131,6 +135,12 @@ rust-postgres-native-tls = { workspace = true, optional = true}
 rumqttc = { workspace = true, optional = true }
 aws-sdk-sqs = { workspace = true, optional = true } 
 aws-config = { workspace = true, optional = true }
+aws-sdk-sts = { workspace = true, optional = true }
 google-cloud-pubsub = { workspace = true, optional = true }
 google-cloud-googleapis = { workspace = true , optional = true }
-tonic = { workspace = true, optional = true }
+tonic = { workspace = true, optional = true }
+deno_error = { workspace = true, optional = true }
+deno_core = { workspace = true, optional = true }
+
+[build-dependencies]
+deno_core = { workspace = true, optional = true }

backend/windmill-api/src/agent_workers_ee.rs

@@ -21,8 +21,8 @@ pub fn workspaced_service(
     _base_internal_url: String,
 ) -> (
     Router,
-    Option<tokio::task::JoinHandle<()>>,
-    windmill_worker::JobCompletedSender,
+    Vec<tokio::task::JoinHandle<()>>,
+    Option<windmill_worker::JobCompletedSender>,
 ) {
     use windmill_common::worker::Connection;
     use windmill_worker::JobCompletedSender;
@@ -32,7 +32,7 @@ pub fn workspaced_service(
 
     let router = Router::new();
 
-    (router, None, job_completed_tx)
+    (router, vec![], Some(job_completed_tx))
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]

backend/windmill-api/src/apps.rs

@@ -20,15 +20,14 @@ use crate::{
 use crate::{
     job_helpers_ee::{
         download_s3_file_internal, get_random_file_name, get_s3_resource,
-        get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req,
-        DownloadFileQuery, LoadImagePreviewQuery,
+        get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery,
     },
     users::fetch_api_authed_from_permissioned_as,
 };
-#[cfg(feature = "parquet")]
 use axum::response::Response;
 use axum::{
-    extract::{Extension, Json, Path, Query},
+    body::Body,
+    extract::{Extension, Json, Multipart, Path, Query},
     response::IntoResponse,
     routing::{delete, get, post},
     Router,
@@ -51,7 +50,6 @@ use sqlx::{types::Uuid, FromRow};
 use std::str;
 use windmill_audit::audit_ee::audit_log;
 use windmill_audit::ActionKind;
-use windmill_common::variables::encrypt;
 use windmill_common::{
     apps::{AppScriptId, ListAppQuery},
     cache::{self, future::FutureCachedExt},
@@ -63,16 +61,24 @@ use windmill_common::{
         http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
         Pagination, StripPath,
     },
-    variables::{build_crypt, build_crypt_with_key_suffix},
+    variables::{build_crypt, build_crypt_with_key_suffix, encrypt},
     worker::{to_raw_value, CLOUD_HOSTED},
     HUB_BASE_URL,
 };
-#[cfg(feature = "parquet")]
-use windmill_common::{jwt, s3_helpers::build_object_store_client};
 
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel};
 
+#[cfg(feature = "parquet")]
+use hmac::Mac;
+#[cfg(feature = "parquet")]
+use windmill_common::{
+    jwt,
+    oauth2::HmacSha256,
+    s3_helpers::{build_object_store_client, S3Object},
+    variables::get_workspace_key,
+};
+
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
@@ -82,10 +88,13 @@ pub fn workspaced_service() -> Router {
         .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
         .route("/get/v/*id", get(get_app_by_id))
+        .route("/get_data/v/*id", get(get_raw_app_data))
         .route("/exists/*path", get(exists_app))
         .route("/update/*path", post(update_app))
+        .route("/update_raw/*path", post(update_app_raw))
         .route("/delete/*path", delete(delete_app))
         .route("/create", post(create_app))
+        .route("/create_raw", post(create_app_raw))
         .route("/history/p/*path", get(get_app_history))
         .route("/get_latest_version/*path", get(get_latest_version))
         .route("/history_update/a/:id/v/:version", post(update_app_history))
@@ -94,6 +103,7 @@ pub fn workspaced_service() -> Router {
             get(list_paths_from_workspace_runnable),
         )
         .route("/custom_path_exists/*custom_path", get(custom_path_exists))
+        .route("/sign_s3_objects", post(sign_s3_objects))
 }
 
 pub fn unauthed_service() -> Router {
@@ -102,10 +112,6 @@ pub fn unauthed_service() -> Router {
         .route("/upload_s3_file/*path", post(upload_s3_file_from_app))
         .route("/delete_s3_file", delete(delete_s3_file_from_app))
         .route("/download_s3_file/*path", get(download_s3_file_from_app))
-        .route(
-            "/load_image_preview/*path",
-            get(load_s3_file_image_preview_from_app),
-        )
         .route("/public_app/:secret", get(get_public_app_by_secret))
         .route("/public_resource/*path", get(get_public_resource))
 }
@@ -132,6 +138,12 @@ pub struct ListableApp {
     #[sqlx(default)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub deployment_msg: Option<String>,
+    #[serde(skip_serializing_if = "is_false")]
+    pub raw_app: bool,
+}
+
+fn is_false(b: &bool) -> bool {
+    !b
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -232,7 +244,8 @@ pub struct S3Input {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct S3Key {
     s3_path: String,
-    resource: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    storage: Option<String>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
@@ -324,7 +337,8 @@ async fn list_apps(
             "app.extra_perms",
             "favorite.path IS NOT NULL as starred",
             "draft.path IS NOT NULL as has_draft",
-            "draft_only"
+            "draft_only",
+            "app_version.raw_app",
         ])
         .left()
         .join("favorite")
@@ -383,6 +397,44 @@ async fn list_apps(
     Ok(Json(rows))
 }
 
+async fn get_raw_app_data(Path((w_id, version_id)): Path<(String, String)>) -> Result<Response> {
+    let file_path = format!("/tmp/wmill/{}/{}", w_id, version_id);
+    let file = tokio::fs::File::open(file_path).await?;
+    let stream = tokio_util::io::ReaderStream::new(file);
+    let res = Response::builder().header(
+        http::header::CONTENT_TYPE,
+        if version_id.ends_with(".css") {
+            "text/css"
+        } else {
+            "text/javascript"
+        },
+    );
+    Ok(res.body(Body::from_stream(stream)).unwrap())
+}
+
+// async fn get_app_version(
+//     authed: ApiAuthed,
+//     Extension(user_db): Extension<UserDB>,
+//     Path((w_id, path)): Path<(String, StripPath)>,
+// ) -> JsonResult<i64> {
+//     let path = path.to_path();
+//     let mut tx = user_db.begin(&authed).await?;
+
+//     let version_o = sqlx::query_scalar!(
+//         "SELECT app.versions[array_upper(app.versions, 1)] as version FROM app
+//             WHERE app.path = $1 AND app.workspace_id = $2",
+//         path,
+//         &w_id,
+//     )
+//     .fetch_optional(&mut *tx)
+//     .await?
+//     .flatten();
+//     tx.commit().await?;
+
+//     let version = not_found_if_none(version_o, "App", path)?;
+//     Ok(Json(version))
+// }
+
 async fn get_app(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -723,6 +775,97 @@ async fn get_secret_id(
     Ok(hx)
 }
 
+macro_rules! process_app_multipart {
+    ($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => {
+        async {
+            let mut saved_app = None;
+            let mut uploaded_js = false;
+
+            //todo: use s3 instead
+            let file_path = format!("/tmp/wmill/{}", $w_id);
+            std::fs::create_dir_all(&file_path).unwrap();
+
+            let mut multipart = $multipart;
+            while let Some(field) = multipart.next_field().await.unwrap() {
+                let name = field.name().unwrap().to_string();
+                let data = field.bytes().await.unwrap();
+                if name == "app" {
+                    let app = serde_json::from_slice(&data).map_err(to_anyhow)?;
+                    let (ntx, npath, nid) = $internal_fn(
+                        $authed.clone(),
+                        $db.clone(),
+                        $user_db.clone(),
+                        $w_id,
+                        $path,
+                        true,
+                        app,
+                    )
+                    .await?;
+                    saved_app = Some((npath, nid, ntx));
+                } else if name == "js" {
+                    if let Some((_npath, id, _tx)) = saved_app.as_ref() {
+                        let file_path = format!("{}/{}.js", file_path, id);
+                        std::fs::write(file_path, data).unwrap();
+                        uploaded_js = true;
+                    } else {
+                        return Err(Error::BadRequest(
+                            "App payload need to be created first".to_string(),
+                        ));
+                    }
+                } else if name == "css" {
+                    if let Some((_npath, id, _tx)) = saved_app.as_ref() {
+                        let file_path = format!("{}/{}.css", file_path, id);
+                        std::fs::write(file_path, data).unwrap();
+                    } else {
+                        return Err(Error::BadRequest(
+                            "App payload need to be created first".to_string(),
+                        ));
+                    }
+                } else {
+                    return Err(Error::BadRequest(format!("Unsupported field: {}", name)));
+                }
+            }
+            if !uploaded_js {
+                return Err(Error::BadRequest("js or css file not uploaded".to_string()));
+            }
+            if let Some((npath, id, tx)) = saved_app {
+                tx.commit().await?;
+                Ok((npath, id))
+            } else {
+                Err(Error::BadRequest("App not created".to_string()))
+            }
+        }
+    };
+}
+
+async fn create_app_raw<'a>(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path(w_id): Path<String>,
+    multipart: Multipart,
+) -> Result<(StatusCode, String)> {
+    let (path, _id) = process_app_multipart!(
+        authed,
+        user_db,
+        db,
+        &w_id,
+        "",
+        multipart,
+        |authed, db, user_db, w_id, _path, raw_app, app| create_app_internal(
+            authed, db, user_db, w_id, raw_app, app
+        )
+    )
+    .await?;
+
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: path.clone() },
+    );
+    Ok((StatusCode::CREATED, path))
+}
+
 async fn list_paths_from_workspace_runnable(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -751,17 +894,36 @@ async fn create_app(
     Extension(db): Extension<DB>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
-    Json(mut app): Json<CreateApp>,
+    Json(app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
-    let mut tx = user_db.clone().begin(&authed).await?;
+    let path = app.path.clone();
+    let (new_tx, _path, _id) = create_app_internal(authed, db, user_db, &w_id, false, app).await?;
 
+    new_tx.commit().await?;
+
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: path.clone() },
+    );
+
+    Ok((StatusCode::CREATED, path))
+}
+
+async fn create_app_internal<'a>(
+    authed: ApiAuthed,
+    db: sqlx::Pool<sqlx::Postgres>,
+    user_db: UserDB,
+    w_id: &String,
+    raw_app: bool,
+    mut app: CreateApp,
+) -> Result<(sqlx::Transaction<'a, sqlx::Postgres>, String, i64)> {
+    let mut tx = user_db.clone().begin(&authed).await?;
     app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
     app.policy.on_behalf_of_email = Some(authed.email.clone());
-
+    let path = app.path.clone();
     if &app.path == "" {
         return Err(Error::BadRequest("App path cannot be empty".to_string()));
     }
-
     let exists = sqlx::query_scalar!(
         "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
         &app.path,
@@ -770,21 +932,19 @@ async fn create_app(
     .fetch_one(&mut *tx)
     .await?
     .unwrap_or(false);
-
     if exists {
         return Err(Error::BadRequest(format!(
             "App with path {} already exists",
             &app.path
         )));
     }
-
     if let Some(custom_path) = &app.custom_path {
         require_admin(authed.is_admin, &authed.username)?;
 
         let exists = sqlx::query_scalar!(
             "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
             custom_path,
-            if *CLOUD_HOSTED { Some(&w_id) } else { None }
+            if *CLOUD_HOSTED { Some(w_id) } else { None }
         )
         .fetch_one(&mut *tx)
         .await?.unwrap_or(false);
@@ -796,7 +956,6 @@ async fn create_app(
             )));
         }
     }
-
     sqlx::query!(
         "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
         &app.path,
@@ -804,7 +963,6 @@ async fn create_app(
     )
     .execute(&mut *tx)
     .await?;
-
     let id = sqlx::query_scalar!(
         "INSERT INTO app
             (workspace_id, path, summary, policy, versions, draft_only, custom_path)
@@ -815,24 +973,24 @@ async fn create_app(
         json!(app.policy),
         app.draft_only,
         app.custom_path
+            .as_ref()
             .map(|s| if s.is_empty() { None } else { Some(s) })
             .flatten()
     )
     .fetch_one(&mut *tx)
     .await?;
-
     let v_id = sqlx::query_scalar!(
         "INSERT INTO app_version
-            (app_id, value, created_by)
-            VALUES ($1, $2::text::json, $3) RETURNING id",
+            (app_id, value, created_by, raw_app)
+            VALUES ($1, $2::text::json, $3, $4) RETURNING id",
         id,
         //to preserve key orders
         serde_json::to_string(&app.value).unwrap(),
         authed.username,
+        raw_app
     )
     .fetch_one(&mut *tx)
     .await?;
-
     sqlx::query!(
         "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE id = $2",
         v_id,
@@ -846,22 +1004,20 @@ async fn create_app(
         &authed,
         "apps.create",
         ActionKind::Create,
-        &w_id,
+        w_id,
         Some(&app.path),
         None,
     )
     .await?;
-
     let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
-    if let Some(dm) = app.deployment_message {
+    if let Some(dm) = &app.deployment_message {
         args.insert("deployment_message".to_string(), to_raw_value(&dm));
     }
-
     let tx = PushIsolationLevel::Transaction(tx);
     let (dependency_job_uuid, new_tx) = push(
         &db,
         tx,
-        &w_id,
+        w_id,
         JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
         PushArgs { args: &args, extra: None },
         &authed.username,
@@ -885,14 +1041,7 @@ async fn create_app(
     .await?;
     tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
 
-    new_tx.commit().await?;
-
-    webhook.send_message(
-        w_id.clone(),
-        WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
-    );
-
-    Ok((StatusCode::CREATED, app.path))
+    Ok((new_tx, path, v_id))
 }
 
 async fn list_hub_apps(Extension(db): Extension<DB>) -> impl IntoResponse {
@@ -1013,12 +1162,76 @@ async fn update_app(
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
 ) -> Result<String> {
-    use sql_builder::prelude::*;
-
+    // create_app_internal(authed, user_db, db, &w_id, &mut app).await?;
     let path = path.to_path();
+    let opath = path.to_string();
+    let (new_tx, npath, _v_id) =
+        update_app_internal(authed, db, user_db, &w_id, path, false, ns).await?;
+    new_tx.commit().await?;
 
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id.clone(),
+            old_path: opath.clone(),
+            new_path: npath.clone(),
+        },
+    );
+
+    Ok(format!("app {} updated (npath: {:?})", opath, npath))
+}
+
+async fn update_app_raw<'a>(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    multipart: Multipart,
+) -> Result<String> {
+    let path = path.to_path();
+    let opath = path.to_string();
+    let (npath, _id) = process_app_multipart!(
+        authed,
+        user_db,
+        db,
+        &w_id,
+        path,
+        multipart,
+        update_app_internal
+    )
+    .await?;
+
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id.clone(),
+            old_path: opath.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
+
+    Ok(format!("app {} updated (npath: {:?})", opath, npath))
+}
+// async fn create_app_internal<'a>(
+//     authed: ApiAuthed,
+//     db: sqlx::Pool<sqlx::Postgres>,
+//     user_db: UserDB,
+//     w_id: &String,
+//     app: &mut CreateApp,
+// )
+
+async fn update_app_internal<'a>(
+    authed: ApiAuthed,
+    db: sqlx::Pool<sqlx::Postgres>,
+    user_db: UserDB,
+    w_id: &str,
+    path: &str,
+    raw_app: bool,
+    ns: EditApp,
+) -> Result<(sqlx::Transaction<'a, sqlx::Postgres>, String, i64)> {
+    use sql_builder::prelude::*;
     let mut tx = user_db.clone().begin(&authed).await?;
-
     let npath = if ns.policy.is_some()
         || ns.path.is_some()
         || ns.summary.is_some()
@@ -1065,7 +1278,7 @@ async fn update_app(
                 let exists = sqlx::query_scalar!(
                     "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
                     ncustom_path,
-                    if *CLOUD_HOSTED { Some(&w_id) } else { None },
+                    if *CLOUD_HOSTED { Some(w_id) } else { None },
                     path,
                     w_id
                 )
@@ -1112,12 +1325,13 @@ async fn update_app(
 
         let v_id = sqlx::query_scalar!(
             "INSERT INTO app_version
-                (app_id, value, created_by)
-                VALUES ($1, $2::text::json, $3) RETURNING id",
+                (app_id, value, created_by, raw_app)
+                VALUES ($1, $2::text::json, $3, $4) RETURNING id",
             app_id,
             //to preserve key orders
             serde_json::to_string(&nvalue).unwrap(),
             authed.username,
+            raw_app
         )
         .fetch_one(&mut *tx)
         .await?;
@@ -1148,7 +1362,6 @@ async fn update_app(
             )));
         }
     };
-
     sqlx::query!(
         "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
         path,
@@ -1156,29 +1369,26 @@ async fn update_app(
     )
     .execute(&mut *tx)
     .await?;
-
     audit_log(
         &mut *tx,
         &authed,
         "apps.update",
         ActionKind::Update,
-        &w_id,
+        w_id,
         Some(&npath),
         None,
     )
     .await?;
-
     let tx = PushIsolationLevel::Transaction(tx);
     let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
     if let Some(dm) = ns.deployment_message {
         args.insert("deployment_message".to_string(), to_raw_value(&dm));
     }
     args.insert("parent_path".to_string(), to_raw_value(&path));
-
     let (dependency_job_uuid, new_tx) = push(
         &db,
         tx,
-        &w_id,
+        w_id,
         JobPayload::AppDependencies { path: npath.clone(), version: v_id },
         PushArgs { args: &args, extra: None },
         &authed.username,
@@ -1201,18 +1411,7 @@ async fn update_app(
     )
     .await?;
     tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
-    new_tx.commit().await?;
-
-    webhook.send_message(
-        w_id.clone(),
-        WebhookMessage::UpdateApp {
-            workspace: w_id,
-            old_path: path.to_owned(),
-            new_path: npath.clone(),
-        },
-    );
-
-    Ok(format!("app {} updated (npath: {:?})", path, npath))
+    Ok((new_tx, npath, v_id))
 }
 
 #[derive(Debug, Deserialize, Clone)]
@@ -1554,15 +1753,97 @@ struct UploadFileToS3Query {
 
 #[cfg(feature = "parquet")]
 #[derive(Serialize, Deserialize)]
-struct DeleteTokenClaims {
+struct S3DeleteTokenClaims {
     file_key: String,
     on_behalf_of_email: String,
     permissioned_as: String,
     username: String,
     s3_resource_path: Option<String>,
+    workspace: String,
     pub exp: usize,
 }
 
+#[cfg(feature = "parquet")]
+#[derive(Deserialize)]
+struct S3TokenRequestBody {
+    s3_objects: Vec<S3Object>,
+}
+#[cfg(feature = "parquet")]
+async fn sign_s3_objects(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(body): Json<S3TokenRequestBody>,
+) -> Result<Json<Vec<S3Object>>> {
+    let workspace_key = get_workspace_key(&w_id, &db).await?;
+
+    let futures = body.s3_objects.into_iter().map(|s3_object| async {
+        let exp = (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp();
+        let mut message = format!("file_key={}&exp={}", s3_object.s3.clone(), exp);
+        if let Some(ref storage) = s3_object.storage {
+            message = format!("{}&storage={}", message, storage);
+        }
+
+        let mut max = HmacSha256::new_from_slice(workspace_key.as_bytes())
+            .map_err(|err| Error::internal_err(format!("Failed to create hmac: {}", err)))?;
+        max.update(message.as_bytes());
+        let result = max.finalize();
+        let signature = hex::encode(result.into_bytes());
+
+        let presigned = format!("exp={}&sig={}", exp, signature);
+
+        Ok::<_, Error>(S3Object { presigned: Some(presigned), ..s3_object })
+    });
+
+    let signed_s3_objects = futures::future::try_join_all(futures).await?;
+
+    Ok(Json(signed_s3_objects))
+}
+
+#[cfg(feature = "parquet")]
+async fn validate_s3_signature(file_query: &AppS3FileQuery, w_id: &str, db: &DB) -> Result<()> {
+    let workspace_key = get_workspace_key(w_id, &db).await?;
+
+    let Some(exp) = file_query
+        .exp
+        .as_ref()
+        .map(|e| e.parse::<i64>().unwrap_or_default())
+    else {
+        return Err(Error::BadRequest("Missing exp".to_string()));
+    };
+
+    let Some(ref sig) = file_query.sig else {
+        return Err(Error::BadRequest("Missing signature".to_string()));
+    };
+
+    let mut message = format!("file_key={}&exp={}", file_query.s3, exp);
+
+    if let Some(ref storage) = file_query.storage {
+        message = format!("{}&storage={}", message, storage);
+    }
+
+    let mut mac = HmacSha256::new_from_slice(workspace_key.as_bytes())
+        .map_err(|err| Error::internal_err(format!("Failed to create hmac: {}", err)))?;
+
+    mac.update(message.as_bytes());
+
+    let sig_bytes = hex::decode(sig)?;
+    mac.verify_slice(&sig_bytes)
+        .map_err(|err| Error::BadRequest(format!("Invalid signature: {}", err)))?;
+
+    if exp < chrono::Utc::now().timestamp() {
+        return Err(Error::BadRequest("Signature expired".to_string()));
+    }
+
+    Ok(())
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn sign_s3_objects() -> Result<()> {
+    return Err(Error::BadRequest(
+        "This endpoint requires the parquet feature to be enabled".to_string(),
+    ));
+}
+
 #[cfg(feature = "parquet")]
 #[derive(Serialize)]
 struct AppUploadFileResponse {
@@ -1817,13 +2098,14 @@ async fn upload_s3_file_from_app(
 
     upload_file_from_req(s3_client, &file_key, request, options).await?;
 
-    let delete_token = jwt::encode_with_internal_secret(DeleteTokenClaims {
+    let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims {
         file_key: file_key.clone(),
         on_behalf_of_email,
         permissioned_as,
         username,
         s3_resource_path: query.s3_resource_path,
-        exp: (chrono::Utc::now() + chrono::Duration::seconds(3600 * 24)).timestamp() as usize,
+        workspace: w_id.clone(),
+        exp: (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp() as usize,
     })
     .await?;
 
@@ -1843,14 +2125,19 @@ async fn delete_s3_file_from_app(
     Path(w_id): Path<String>,
     Query(query): Query<DeleteS3FileQuery>,
 ) -> Result<()> {
-    let DeleteTokenClaims {
+    let S3DeleteTokenClaims {
         file_key,
         on_behalf_of_email,
         permissioned_as,
         username,
         s3_resource_path,
+        workspace,
         ..
-    } = jwt::decode_with_internal_secret::<DeleteTokenClaims>(&query.delete_token).await?;
+    } = jwt::decode_with_internal_secret::<S3DeleteTokenClaims>(&query.delete_token).await?;
+
+    if workspace != w_id {
+        return Err(Error::BadRequest("Invalid workspace".to_string()));
+    }
 
     let on_behalf_authed = fetch_api_authed_from_permissioned_as(
         permissioned_as,
@@ -1958,7 +2245,7 @@ async fn get_on_behalf_authed_from_app(
 async fn check_if_allowed_to_access_s3_file_from_app(
     db: &DB,
     opt_authed: &Option<ApiAuthed>,
-    file_key: &str,
+    file_query: &AppS3FileQuery,
     w_id: &str,
     path: &str,
     policy: &Policy,
@@ -1966,40 +2253,59 @@ async fn check_if_allowed_to_access_s3_file_from_app(
     // if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours
     // otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy)
 
-    let allowed = opt_authed.is_some()
-        || sqlx::query_scalar!(
-            r#"SELECT EXISTS (
-                SELECT 1 FROM v2_as_completed_job 
-                WHERE workspace_id = $2 
-                    AND (job_kind = 'appscript' OR job_kind = 'preview')
-                    AND created_by = 'anonymous' 
-                    AND started_at > now() - interval '3 hours'
-                    AND script_path LIKE $3 || '/%' 
-                    AND result @> ('{"s3":"' || $1 ||  '"}')::jsonb 
-            )"#,
-            file_key,
-            w_id,
-            path,
-        )
-        .fetch_one(db)
-        .await?
-        .unwrap_or(false)
-
-        // check if the file is allowed by the allowed_s3_keys policy
-        || policy.allowed_s3_keys.as_ref().unwrap().iter().any(|key| key.s3_path == file_key);
-
-    if !allowed {
-        Err(Error::BadRequest("File restricted".to_string()))
-    } else {
+    if file_query.sig.is_some() {
+        validate_s3_signature(file_query, w_id, &db).await
+    } else if opt_authed.is_some() {
         Ok(())
+    } else {
+        let allowed = policy
+            .allowed_s3_keys
+            .as_ref()
+            .unwrap()
+            .iter()
+            .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage)
+            || {
+                sqlx::query_scalar!(
+                    r#"SELECT EXISTS (
+                SELECT 1 FROM v2_as_completed_job
+                WHERE workspace_id = $2
+                    AND (job_kind = 'appscript' OR job_kind = 'preview')
+                    AND created_by = 'anonymous'
+                    AND started_at > now() - interval '3 hours'
+                    AND script_path LIKE $3 || '/%'
+                    AND result @> ('{"s3":"' || $1 ||  '"}')::jsonb
+            )"#,
+                    file_query.s3,
+                    w_id,
+                    path,
+                )
+                .fetch_one(db)
+                .await?
+                .unwrap_or(false)
+            };
+
+        if !allowed {
+            Err(Error::BadRequest("File restricted".to_string()))
+        } else {
+            Ok(())
+        }
     }
 }
 
 #[cfg(feature = "parquet")]
-#[derive(Deserialize)]
-pub struct DownloadFileQueryWithForceViewerAllowedS3Keys {
+#[derive(Deserialize, Debug)]
+struct AppS3FileQuery {
+    s3: String,
+    storage: Option<String>,
+    sig: Option<String>,
+    exp: Option<String>,
+}
+
+#[cfg(feature = "parquet")]
+#[derive(Deserialize, Debug)]
+struct AppS3FileQueryWithForceViewerAllowedS3Keys {
     #[serde(flatten)]
-    pub file_query: DownloadFileQuery,
+    pub file_query: AppS3FileQuery,
     pub force_viewer_allowed_s3_keys: Option<String>,
 }
 
@@ -2008,7 +2314,7 @@ async fn download_s3_file_from_app(
     OptAuthed(opt_authed): OptAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
-    Query(query): Query<DownloadFileQueryWithForceViewerAllowedS3Keys>,
+    Query(query): Query<AppS3FileQueryWithForceViewerAllowedS3Keys>,
 ) -> Result<Response> {
     let path = path.to_path();
 
@@ -2027,46 +2333,26 @@ async fn download_s3_file_from_app(
     check_if_allowed_to_access_s3_file_from_app(
         &db,
         &opt_authed,
-        &query.file_query.file_key,
+        &query.file_query,
         &w_id,
         &path,
         &policy,
     )
     .await?;
 
-    download_s3_file_internal(on_behalf_authed, &db, None, "", &w_id, query.file_query).await
-}
-
-#[cfg(not(feature = "parquet"))]
-async fn load_s3_file_image_preview_from_app() -> Result<()> {
-    return Err(Error::BadRequest(
-        "This endpoint requires the parquet feature to be enabled".to_string(),
-    ));
-}
-
-#[cfg(feature = "parquet")]
-async fn load_s3_file_image_preview_from_app(
-    OptAuthed(opt_authed): OptAuthed,
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
-    Query(query): Query<LoadImagePreviewQuery>,
-) -> Result<Response> {
-    let path = path.to_path();
-
-    let (on_behalf_authed, policy) =
-        get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, None).await?;
-
-    check_if_allowed_to_access_s3_file_from_app(
+    download_s3_file_internal(
+        on_behalf_authed,
         &db,
-        &opt_authed,
-        &query.file_key,
+        None,
+        "",
         &w_id,
-        &path,
-        &policy,
+        DownloadFileQuery {
+            file_key: query.file_query.s3,
+            s3_resource_path: None,
+            storage: query.file_query.storage,
+        },
     )
-    .await?;
-
-    load_image_preview_internal(on_behalf_authed, &db, "", &w_id, query).await
+    .await
 }
 
 fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {

backend/windmill-api/src/auth.rs

@@ -511,6 +511,13 @@ where
             let path_vec: Vec<&str> = original_uri.path().split("/").collect();
             let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
                 Some(path_vec[3].to_owned())
+            } else if path_vec.len() >= 5
+                && path_vec[0] == ""
+                && path_vec[1] == "api"
+                && path_vec[2] == "mcp"
+                && path_vec[3] == "w"
+            {
+                Some(path_vec[4].to_string())
             } else {
                 if path_vec.len() >= 5
                     && path_vec[0] == ""

backend/windmill-api/src/capture.rs

@@ -18,16 +18,22 @@ use {
 
 #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
 use crate::gcp_triggers_ee::{
-    manage_google_subscription, process_google_push_request, validate_jwt_token, SubscriptionMode,
+    manage_google_subscription, process_google_push_request, validate_jwt_token,
+    CreateUpdateConfig, SubscriptionMode,
 };
 
+#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
+use windmill_common::auth::aws::AwsAuthResourceType;
+
 #[cfg(any(
     feature = "http_trigger",
     all(feature = "enterprise", feature = "gcp_trigger")
 ))]
 use {
-    axum::extract::Request, http::HeaderMap, serde::de::DeserializeOwned,
-    windmill_common::error::Error,
+    axum::extract::Request,
+    http::HeaderMap,
+    serde::de::DeserializeOwned,
+    windmill_common::{error::Error, utils::empty_string_as_none},
 };
 
 #[cfg(all(feature = "enterprise", feature = "kafka"))]
@@ -150,14 +156,20 @@ pub struct SqsTriggerConfig {
     pub queue_url: String,
     pub aws_resource_path: String,
     pub message_attributes: Option<Vec<String>>,
+    pub aws_auth_resource_type: AwsAuthResourceType,
 }
 
 #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
 #[derive(Debug, Serialize, Deserialize)]
 pub struct GcpTriggerConfig {
     pub gcp_resource_path: String,
-    #[serde(flatten)]
     pub subscription_mode: SubscriptionMode,
+    #[serde(default, deserialize_with = "empty_string_as_none")]
+    pub subscription_id: Option<String>,
+    #[serde(default, deserialize_with = "empty_string_as_none")]
+    pub base_endpoint: Option<String>,
+    #[serde(flatten)]
+    pub create_update: Option<CreateUpdateConfig>,
     pub topic_id: String,
 }
 
@@ -366,11 +378,15 @@ async fn set_gcp_trigger_config(
         &gcp_config.gcp_resource_path,
         &capture_config.path,
         &gcp_config.topic_id,
+        &mut gcp_config.subscription_id,
+        &mut gcp_config.base_endpoint,
         gcp_config.subscription_mode,
+        gcp_config.create_update,
+        false,
     )
     .await?;
-
-    gcp_config.subscription_mode = SubscriptionMode::CreateUpdate(config);
+    gcp_config.create_update = Some(config);
+    gcp_config.subscription_mode = SubscriptionMode::CreateUpdate;
     capture_config.trigger_config = Some(TriggerConfig::Gcp(gcp_config));
 
     Ok(capture_config)
@@ -887,7 +903,7 @@ async fn gcp_payload(
 
     let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?;
 
-    let SubscriptionMode::CreateUpdate(config) = &gcp_trigger_config.subscription_mode else {
+    let Some(config) = &gcp_trigger_config.create_update else {
         return Err(Error::BadConfig("Bad config".to_string()));
     };
 

backend/windmill-api/src/embeddings.rs

@@ -253,7 +253,7 @@ impl ModelInstance {
             let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?;
             let token_type_ids = token_ids.zeros_like()?;
 
-            let embedding = self.model.forward(&token_ids, &token_type_ids)?;
+            let embedding = self.model.forward(&token_ids, &token_type_ids, None)?;
             let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?;
             let embedding = normalize_l2(&embedding)?;
 

backend/windmill-api/src/gcp_triggers_ee.rs

@@ -52,11 +52,12 @@ pub struct ExistingGcpSubscription {
     pub base_endpoint: String,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(tag = "subscription_mode", rename_all = "snake_case")]
+#[derive(Debug, Deserialize, Serialize, sqlx::Type)]
+#[serde(rename_all = "snake_case")]
+#[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")]
 pub enum SubscriptionMode {
-    Existing(ExistingGcpSubscription),
-    CreateUpdate(CreateUpdateConfig),
+    Existing,
+    CreateUpdate,
 }
 
 pub fn workspaced_service() -> Router {
@@ -77,7 +78,11 @@ pub async fn manage_google_subscription(
     _gcp_resource_path: &str,
     _path: &str,
     _topic_id: &str,
+    _subscription_id: &mut Option<String>,
+    _base_endpoint: &mut Option<String>,
     _subscription_mode: SubscriptionMode,
+    _create_update_config: Option<CreateUpdateConfig>,
+    _trigger_mode: bool,
 ) -> WindmillResult<CreateUpdateConfig> {
     Ok(CreateUpdateConfig::default())
 }
@@ -117,6 +122,7 @@ pub struct GcpTrigger {
     pub subscription_id: String,
     pub delivery_type: DeliveryType,
     pub delivery_config: Option<SqlxJson<PushConfig>>,
+    pub subscription_mode: SubscriptionMode,
     pub topic_id: String,
     pub path: String,
     pub script_path: String,

backend/windmill-api/src/http_trigger_auth.rs

@@ -253,8 +253,8 @@ mod zoom {
     use super::*;
 
     #[derive(Debug, Deserialize)]
-    #[serde(rename_all = "snake_case")]
     struct ZoomPayload {
+        #[serde(rename = "plainToken")]
         plain_token: String,
     }
 

backend/windmill-api/src/job_helpers_ee.rs

@@ -30,12 +30,20 @@ pub struct UploadFileResponse {
 
 #[derive(Deserialize)]
 pub struct LoadImagePreviewQuery {
+    #[allow(dead_code)]
     pub file_key: String,
+    #[allow(dead_code)]
+    pub storage: Option<String>,
 }
 
 #[derive(Deserialize)]
 pub struct DownloadFileQuery {
+    #[allow(dead_code)]
     pub file_key: String,
+    #[allow(dead_code)]
+    pub storage: Option<String>,
+    #[allow(dead_code)]
+    pub s3_resource_path: Option<String>,
 }
 
 pub fn workspaced_service() -> Router {
@@ -111,16 +119,3 @@ pub async fn download_s3_file_internal(
         "Not implemented in Windmill's Open Source repository".to_string(),
     ))
 }
-
-#[cfg(feature = "parquet")]
-pub async fn load_image_preview_internal(
-    _authed: ApiAuthed,
-    _db: &DB,
-    _token: &str,
-    _w_id: &str,
-    _query: LoadImagePreviewQuery,
-) -> error::Result<Response> {
-    Err(error::Error::internal_err(
-        "Not implemented in Windmill's Open Source repository".to_string(),
-    ))
-}