chore(deps): bump actions/setup-python from 2 to 5

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 5.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v2...v5)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/backend-check.yml

@@ -0,0 +1,134 @@
+name: Backend check
+on:
+  push:
+    paths:
+        - "backend/**"
+        - ".github/workflows/backend-check.yml"
+
+jobs:
+  check_oss:
+    runs-on: ubicloud-standard-8
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: backend
+          toolchain: 1.82.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
+      - name: cargo check
+        working-directory: ./backend
+        timeout-minutes: 16
+        run: SQLX_OFFLINE=true cargo check
+
+  check_oss_full:
+    runs-on: ubicloud-standard-8
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+
+      - name: install xmlsec1
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y libxml2-dev libxmlsec1-dev 
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: backend
+          toolchain: 1.82.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
+      - name: cargo check
+        working-directory: ./backend
+        timeout-minutes: 16
+        run: |
+          mkdir -p fake_frontend_build
+          FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features
+
+  check_ee:
+    runs-on: ubicloud-standard-8
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: 0
+
+
+      - name: Substitute EE code (EE logic is behind feature flag)
+        run: |
+          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: backend
+          toolchain: 1.82.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
+      - name: cargo check
+        working-directory: ./backend
+        timeout-minutes: 16
+        run: SQLX_OFFLINE=true cargo check
+
+  check_ee_full:
+    runs-on: ubicloud-standard-8
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: 0
+
+      - name: install xmlsec1
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y libxml2-dev libxmlsec1-dev 
+
+      - name: Substitute EE code (EE logic is behind feature flag)
+        run: |
+          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: backend
+          toolchain: 1.82.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
+      - name: cargo check
+        timeout-minutes: 16
+        working-directory: ./backend
+        run: |
+          mkdir -p fake_frontend_build
+          FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features
+
+ 

.github/workflows/backend-test.yml

@@ -13,40 +13,58 @@ on:
       - "backend/**"
       - ".github/workflows/backend-test.yml"
 
+defaults:
+  run:
+    working-directory: ./backend
+
 jobs:
   cargo_test:
     runs-on: ubicloud-standard-8
-    container:
-      image: ghcr.io/windmill-labs/backend-tests
     services:
       postgres:
         image: postgres
+        ports:
+          - 5432:5432
         env:
           POSTGRES_DB: windmill
           POSTGRES_PASSWORD: changeme
-
         options: >-
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: "9.0.x"
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.21.5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.1.40
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.4.18"
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: 1.80.0
-      # - uses: Swatinem/rust-cache@v2
-      #   with:
-      #     workspaces: |
-      #       backend
-      #       backend -> target
+          cache-workspaces: backend
+          toolchain: 1.82.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
       - name: cargo test
-        timeout-minutes: 15
+        timeout-minutes: 16
         run:
-          /usr/bin/deno --version &&
-          /usr/bin/bun -v &&
-          go version &&
-          /usr/local/bin/python3 --version &&
-          mkdir frontend/build && cd backend && touch
-          windmill-api/openapi-deref.yaml &&
-          DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill
-          DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features
-          enterprise,deno_core --all -- --nocapture
+          deno --version && bun -v && go version && python3 --version &&
+          SQLX_OFFLINE=true
+          DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill
+          DISABLE_EMBEDDING=true RUST_LOG=info PYTHON_PATH=$(which python)
+          DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go)
+          UV_PATH=$(which uv) cargo test --features
+          enterprise,deno_core,license,python,rust,scoped_cache --all -- --nocapture

.github/workflows/build-publish-rh-image.yml

@@ -64,7 +64,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,http_trigger,zip,oauth2,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
           secrets: |
             rh_username=${{ secrets.RH_USERNAME }}
             rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
           platforms: linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,http_trigger,zip,oauth2,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
           secrets: |
             rh_username=${{ secrets.RH_USERNAME }}
             rh_password=${{ secrets.RH_PASSWORD }}
@@ -111,12 +111,16 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: RHEL9-amd64 build
-          path: ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel9
+          path:
+            ${{ steps.extract-ee-amd64.outputs.destination
+            }}/windmill-ee-amd64-rhel9
 
       - uses: actions/upload-artifact@v4
         with:
           name: RHEL9-arm64 build
-          path: ${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel9
+          path:
+            ${{ steps.extract-ee-arm64.outputs.destination
+            }}/windmill-ee-arm64-rhel9
 
       # - name: Attach binary to release
       #   uses: softprops/action-gh-release@v2

.github/workflows/build-staging-image.yml

@@ -1,70 +0,0 @@
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
-name: Build windmill-staging
-on:
-  workflow_dispatch:
-
-permissions: write-all
-
-jobs:
-  build_ee:
-    runs-on: ubicloud
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Read EE repo commit hash
-        run: |
-          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@v4
-        with:
-          repository: windmill-labs/windmill-ee-private
-          path: ./windmill-ee-private
-          ref: ${{ env.ee_repo_ref }}
-          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
-          fetch-depth: 0
-
-      # - name: Set up Docker Buildx
-      #   uses: docker/setup-buildx-action@v2
-      - uses: depot/setup-action@v1
-
-      - name: Docker meta
-        id: meta-ee-public
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-staging-ee
-          flavor: |
-            latest=false
-          tags: |
-            type=sha
-            type=ref,event=branch
-
-      - name: Login to registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Substitute EE code
-        run: |
-          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
-
-      - name: Build and push publicly ee
-        uses: depot/build-push-action@v1
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp
-          tags: |
-            ${{ steps.meta-ee-public.outputs.tags }}
-          labels: |
-            ${{ steps.meta-ee-public.outputs.labels }}
-            org.opencontainers.image.licenses=Windmill-Enterprise-License

.github/workflows/build_windows_worker_.yml

@@ -45,7 +45,7 @@ jobs:
           $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
           mkdir frontend/build && cd backend
           New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
-          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp
+          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
 
       - name: Rename binary with corresponding architecture
         run: |

.github/workflows/build_ws.yml

@@ -19,7 +19,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: depot/setup-action@v1
+      # - uses: depot/setup-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -40,7 +43,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push publicly
-        uses: depot/build-push-action@v1
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./docker/DockerfileMultiplayer

.github/workflows/docker-image-rpi4.yml

@@ -67,7 +67,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,deno_core,php,mysql,csharp
+            features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
             ${{ steps.meta-public.outputs.tags }}

.github/workflows/docker-image.yml

@@ -1,10 +1,9 @@
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository ||
-    'windmill-labs/windmill-test' }}
-  DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
-    github.event.number) }}
-
+  IMAGE_NAME:
+    ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' && github.repository || 'windmill-labs/windmill-test' }}
+  DEV_SHA:
+    ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' && 'dev' || github.event.inputs.tag || github.sha }}
 name: Build windmill:main
 on:
   push:
@@ -14,6 +13,17 @@ on:
     types: [opened, synchronize, reopened]
     paths:
       - "Dockerfile"
+  workflow_dispatch:
+    inputs:
+      ee:
+        description: 'Build EE image (true, false)'
+        required: false
+        default: false
+        type: boolean
+      tag:
+        description: 'Tag the image'
+        required: true
+        default: 'test'
 
 concurrency:
   group: ${{ github.ref }}
@@ -24,7 +34,7 @@ permissions: write-all
 jobs:
   build:
     runs-on: ubicloud
-    if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_all') || contains(github.event.comment.body, '/buildimage_base'))
+    if: (github.event_name != 'workflow_dispatch') || (github.event.inputs && !github.event.inputs.ee)
     steps:
       - uses: actions/checkout@v4
         with:
@@ -76,7 +86,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql,csharp
+            features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
             ${{ steps.meta-public.outputs.tags }}
@@ -86,7 +96,8 @@ jobs:
 
   build_ee:
     runs-on: ubicloud
-    if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_ee') || contains(github.event.comment.body, '/buildimage_nsjail')) || contains(github.event.comment.body, '/buildimage_all')
+    if:
+      (github.event_name != 'workflow_dispatch') || (github.event.inputs.ee)
     steps:
       - uses: actions/checkout@v4
         with:
@@ -138,7 +149,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
             ${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +211,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
             PYTHON_IMAGE=python:3.12.2-slim-bookworm
           tags: |
             ${{ steps.meta-ee-public-py312.outputs.tags }}
@@ -356,7 +367,9 @@ jobs:
   tag_latest:
     runs-on: ubicloud
     needs: [run_integration_test, build]
-    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    if:
+      github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' ||
+      startsWith(github.ref, 'refs/tags/v'))
     steps:
       - uses: actions/checkout@v4
         with:
@@ -375,7 +388,9 @@ jobs:
   tag_latest_ee:
     runs-on: ubicloud
     needs: [run_integration_test, build_ee]
-    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    if:
+      github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' ||
+      startsWith(github.ref, 'refs/tags/v'))
     steps:
       - uses: actions/checkout@v4
         with:
@@ -436,7 +451,8 @@ jobs:
   build_ee_nsjail:
     needs: [build_ee]
     runs-on: ubicloud
-    if: (github.event_name != 'issue_comment') || (github.event_name != 'pull_request') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all'))
+    if:
+      (github.event_name != 'pull_request') && (github.event_name != 'workflow_dispatch')
     steps:
       - uses: actions/checkout@v4
         with:
@@ -529,7 +545,7 @@ jobs:
   publish_ecr_s3:
     needs: [build_ee_nsjail]
     runs-on: ubicloud-standard-2-arm
-    if: github.event_name != 'pull_request'
+    if: (github.event_name != 'pull_request') && (github.event_name != 'workflow_dispatch')
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/publish_windows_worker.yml

@@ -47,7 +47,7 @@ jobs:
           $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
           mkdir frontend/build && cd backend
           New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
-          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp
+          cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp,static_frontend,rust
 
       - name: Rename binary with corresponding architecture
         run: |

CHANGELOG.md

@@ -1,5 +1,58 @@
 # Changelog
 
+## [1.441.0](https://github.com/windmill-labs/windmill/compare/v1.440.3...v1.441.0) (2024-12-20)
+
+
+### Features
+
+* interactive slack approvals ([#4942](https://github.com/windmill-labs/windmill/issues/4942)) ([6308bf0](https://github.com/windmill-labs/windmill/commit/6308bf0dcb1d6670e839a1a1e0b794bf3ce6520c))
+
+## [1.440.3](https://github.com/windmill-labs/windmill/compare/v1.440.2...v1.440.3) (2024-12-19)
+
+
+### Bug Fixes
+
+* update bun from 1.1.38 to 1.1.40 ([c4fdd22](https://github.com/windmill-labs/windmill/commit/c4fdd2297efc43ce557cc9791151301377126c29))
+
+## [1.440.2](https://github.com/windmill-labs/windmill/compare/v1.440.1...v1.440.2) (2024-12-18)
+
+
+### Bug Fixes
+
+* fix redeploying flows with attached schedules ([fb536df](https://github.com/windmill-labs/windmill/commit/fb536df0668d49d14f4aed98870caaad396d0389))
+
+## [1.440.1](https://github.com/windmill-labs/windmill/compare/v1.440.0...v1.440.1) (2024-12-18)
+
+
+### Bug Fixes
+
+* **internal:** updating rust to 1.82 ([02a8f1f](https://github.com/windmill-labs/windmill/commit/02a8f1f86453a5f8769364ba3798998b8830d086))
+
+## [1.440.0](https://github.com/windmill-labs/windmill/compare/v1.439.0...v1.440.0) (2024-12-18)
+
+
+### Features
+
+* **cache:** remove persistent raw values from queue ([#4866](https://github.com/windmill-labs/windmill/issues/4866)) ([977ac5c](https://github.com/windmill-labs/windmill/commit/977ac5c3f3c2e8224f4915e483ae60f28ce008fc))
+
+
+### Bug Fixes
+
+* add workspace selector and fix css for create webhook page ([#4939](https://github.com/windmill-labs/windmill/issues/4939)) ([d880655](https://github.com/windmill-labs/windmill/commit/d8806555f1d78a5fae7ae77acbcdad402e89951d))
+* fix relative imports in cached flow scripts ([13be0cd](https://github.com/windmill-labs/windmill/commit/13be0cd1c822d7a809dc96914dd1286510b9f9eb))
+
+## [1.439.0](https://github.com/windmill-labs/windmill/compare/v1.438.0...v1.439.0) (2024-12-15)
+
+
+### Features
+
+* add multipart/form-data support ([#4927](https://github.com/windmill-labs/windmill/issues/4927)) ([83a60cb](https://github.com/windmill-labs/windmill/commit/83a60cbc517d5ddab24b247fd5e452175d59ad07))
+
+
+### Bug Fixes
+
+* ECS terraform db url + ami issues ([#4924](https://github.com/windmill-labs/windmill/issues/4924)) ([5172c13](https://github.com/windmill-labs/windmill/commit/5172c13ab8e9aeb1a83c161e7e7f63ebfb40b008))
+
 ## [1.438.0](https://github.com/windmill-labs/windmill/compare/v1.437.1...v1.438.0) (2024-12-13)
 
 

Dockerfile

@@ -1,5 +1,5 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
-ARG RUST_IMAGE=rust:1.80-slim-bookworm
+ARG RUST_IMAGE=rust:1.82-slim-bookworm
 ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
@@ -177,7 +177,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
 
-COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.40 /usr/local/bin/bun /usr/bin/bun
 
 COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
 COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer

backend/.sqlx/query-0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213.json

@@ -0,0 +1,67 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                job_kind AS \"job_kind: JobKind\",\n                script_hash AS \"script_hash: ScriptHash\",\n                flow_status AS \"flow_status!: Json<Box<RawValue>>\",\n                raw_flow AS \"raw_flow: Json<Box<RawValue>>\"\n            FROM queue WHERE id = $1 AND workspace_id = $2 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_kind: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "script_hash: ScriptHash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "flow_status!: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 3,
+        "name": "raw_flow: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "0a6a89e6ab3037f02c3c4c84ee02138d5fded1e6360bb992046fe9711b5ea213"
+}

backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json

@@ -75,7 +75,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-0bf123446bebbc357c58a53a9319f4954dbf3225e91cbe999e5b264c1a747664.json

@@ -0,0 +1,72 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT content AS \"content!: String\",\n                    lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "content!: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "language: Option<ScriptLang>",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "envs: Vec<String>",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 4,
+        "name": "codebase: String",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "0bf123446bebbc357c58a53a9319f4954dbf3225e91cbe999e5b264c1a747664"
+}

backend/.sqlx/query-10cd655d3d2916721b530c7faa7652fb8fae25383b58f6b9e8dc431b76947315.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value AS \"value!: Json<Box<RawValue>>\"\n                    FROM flow_version WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value!: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "10cd655d3d2916721b530c7faa7652fb8fae25383b58f6b9e8dc431b76947315"
+}

backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json

@@ -52,7 +52,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-2b2ee874dbd90beec26713d2effdc5d011d9f1091a13761642d064220add7b41.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"\n                    FROM flow WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version!: i64",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "2b2ee874dbd90beec26713d2effdc5d011d9f1091a13761642d064220add7b41"
+}

backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json

@@ -61,7 +61,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce.json

@@ -0,0 +1,73 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            script_path, script_hash AS \"script_hash: ScriptHash\",\n            job_kind AS \"job_kind: JobKind\",\n            flow_status AS \"flow_status: Json<Box<RawValue>>\",\n            raw_flow AS \"raw_flow: Json<Box<RawValue>>\"\n        FROM completed_job WHERE id = $1 and workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "script_hash: ScriptHash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "job_kind: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "flow_status: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 4,
+        "name": "raw_flow: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "3df03ec2345c905f03450e1e3f0c3ce7f41a22e72c5180ef8cc910305e4d0fce"
+}

backend/.sqlx/query-3ee2d60ca93eeaf02ab0ee96aca399ec055b044a06284c0ab19b67d97f803894.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT code AS \"raw_code: String\", lock AS \"raw_lock: String\", flow AS \"raw_flow: Json<Box<RawValue>>\" FROM flow_node WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "raw_code: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "raw_lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "raw_flow: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "3ee2d60ca93eeaf02ab0ee96aca399ec055b044a06284c0ab19b67d97f803894"
+}

backend/.sqlx/query-497d93db931922b09f96cf73239513d7141f3d37f85ada46597079991b3bff30.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT \n            MAX (created_at) as last_deploy, \n          COUNT (*) as deploys_count \n           FROM metrics \n          WHERE id = 'no_uv_usage_py'",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "last_deploy",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 1,
-        "name": "deploys_count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null,
-      null
-    ]
-  },
-  "hash": "497d93db931922b09f96cf73239513d7141f3d37f85ada46597079991b3bff30"
-}

backend/.sqlx/query-4a804ee30bfe86c4e2c15a9f6511be5adf0dd22cb942fac64b439fb4e20df447.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                        INSERT INTO metrics (id, value) \n                            VALUES ('no_uv_usage_ansible', $1)\n                        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4a804ee30bfe86c4e2c15a9f6511be5adf0dd22cb942fac64b439fb4e20df447"
+}

backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json

@@ -55,7 +55,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-700f987ec9b2a17c9b8304d598269f8fb58a432934163588af5ea84481c1c087.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND\n                    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "700f987ec9b2a17c9b8304d598269f8fb58a432934163588af5ea84481c1c087"
+}

backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                      INSERT INTO metrics (id, value) \n                           VALUES ('no_uv_usage_py', $1)\n                    ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f"
-}

backend/.sqlx/query-7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value AS \"value!: Json<Box<RawValue>>\"\n                FROM flow_version WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value!: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "7a9a711c7cb05ed8a61116586fb68ef270567ebd70266f07139f3e3e34940699"
+}

backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json

@@ -30,7 +30,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-85085cdc08c262c4750566a0a0b9754017b890ea8c1161a5892b6c29e663ee0e.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value AS \"value!: Json<Box<RawValue>>\"\n                    FROM flow_version_lite WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value!: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "85085cdc08c262c4750566a0a0b9754017b890ea8c1161a5892b6c29e663ee0e"
+}

backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"\n                FROM flow WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version!: i64",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1"
+}

backend/.sqlx/query-9c1a1d0feb79f750c7143fabb0cfa7dab8dd683cd294c27d0549bd8d78ab60a0.json

@@ -0,0 +1,71 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT lock AS \"lock: String\", content AS \"code!: String\",\n                            language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "code!: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "language: Option<ScriptLang>",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "envs: Vec<String>",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 4,
+        "name": "codebase: String",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "9c1a1d0feb79f750c7143fabb0cfa7dab8dd683cd294c27d0549bd8d78ab60a0"
+}

backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -55,7 +55,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-9ee2f67042c1bed1e7d13eb7d07e78991e5d7cf01fc7993531dbedf33dac2e0b.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT lock AS \"lock: String\", code AS \"code: String\", flow AS \"flow: Json<Box<RawValue>>\" FROM flow_node WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "code: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "flow: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "9ee2f67042c1bed1e7d13eb7d07e78991e5d7cf01fc7993531dbedf33dac2e0b"
+}

backend/.sqlx/query-a1f8667bfd5b000dd7ba384e78f2bd7fabb6b8055f4559f77e20eef7c2b1c902.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                      INSERT INTO metrics (id, value) \n                           VALUES ('no_uv_usage_py', ''::text::jsonb)\n                    ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "a1f8667bfd5b000dd7ba384e78f2bd7fabb6b8055f4559f77e20eef7c2b1c902"
-}

backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json

@@ -38,7 +38,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-b7f2ed32e933b65fa5455928c71f61068ad7dfee8352a76122f5af893ffe6517.json

@@ -0,0 +1,72 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT content AS \"content!: String\",\n                        lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "content!: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "language: Option<ScriptLang>",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "envs: Vec<String>",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 4,
+        "name": "codebase: String",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "b7f2ed32e933b65fa5455928c71f61068ad7dfee8352a76122f5af893ffe6517"
+}

backend/.sqlx/query-bb6141ad0e93986b38ccdf4d027c486137d4fe79906e22842b711f4a9379b8c8.json

@@ -0,0 +1,67 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            queue.job_kind AS \"job_kind: JobKind\",\n            queue.script_hash AS \"script_hash: ScriptHash\",\n            queue.raw_flow AS \"raw_flow: sqlx::types::Json<Box<RawValue>>\",\n            completed_job.parent_job AS \"parent_job: Uuid\"\n        FROM queue\n        JOIN completed_job ON completed_job.parent_job = queue.id\n        WHERE completed_job.id = $1 AND completed_job.workspace_id = $2\n        LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "job_kind: JobKind",
+        "type_info": {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow",
+                "flowscript",
+                "flownode",
+                "appscript"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "script_hash: ScriptHash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "raw_flow: sqlx::types::Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 3,
+        "name": "parent_job: Uuid",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "bb6141ad0e93986b38ccdf4d027c486137d4fe79906e22842b711f4a9379b8c8"
+}

backend/.sqlx/query-bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json<Box<RawValue>>\" FROM job WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "raw_code",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "raw_lock",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "raw_flow: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "bc8ac03254669951654cda4bcfa12491341e745aef5e0e5090c2f4e4a4dc54fb"
+}

backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT true FROM token WHERE token = $1 and expiration IS NOT NULL and expiration > now() + $2::int * '1 sec'::interval",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0"
+}

backend/.sqlx/query-c4ee16065fa021cf6443c2f29f1ca986b9730eae9df26dd33c3ea21e35df62c4.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO resource\n    (workspace_id, path, value, resource_type, created_by, edited_at)\n    VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n    DO UPDATE SET value = $3, edited_at = now()",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Jsonb",
-        "Varchar",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "c4ee16065fa021cf6443c2f29f1ca986b9730eae9df26dd33c3ea21e35df62c4"
-}

backend/.sqlx/query-c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value AS \"value!: Json<Box<RawValue>>\"\n                FROM flow_version_lite WHERE id = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value!: Json<Box<RawValue>>",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "c7de95ef4934752b62d6d01bf663ab6a104bbcfb827a4a9e4e03e38732375d55"
+}

backend/.sqlx/query-d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d.json

@@ -0,0 +1,72 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT lock AS \"lock: String\", content AS \"code!: String\",\n                        language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "lock: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "code!: String",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "language: Option<ScriptLang>",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible",
+                "csharp"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "envs: Vec<String>",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 4,
+        "name": "codebase: String",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      true,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "d061e7ca73987036928e17245360fd7f2969aab0c422a00bbdade6a7236aa75d"
+}

backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                      INSERT INTO metrics (id, value) \n                           VALUES ('no_uv_usage_ansible', $1)\n                    ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052"
-}

backend/.sqlx/query-df52a71d59eb84a2b08133d25f0a8bba7f2b56625fdb7a7c0e10b51377eeb1d4.json

@@ -37,7 +37,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-eb125df0f64c0caa07f50dc82e8ae9c6cd2872c7afe96d41bed731c5041ab671.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT \n            MAX (created_at) as last_deploy, \n          COUNT (*) as deploys_count \n           FROM metrics \n          WHERE id = 'no_uv_usage_ansible'",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "last_deploy",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 1,
-        "name": "deploys_count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      null,
-      null
-    ]
-  },
-  "hash": "eb125df0f64c0caa07f50dc82e8ae9c6cd2872c7afe96d41bed731c5041ab671"
-}

backend/.sqlx/query-ed318070b26861fda2d591a4356fdbeb6c7fdc965be43bddb010fd8299af1286.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                          INSERT INTO metrics (id, value) \n                               VALUES ('no_uv_usage_py', $1)\n                        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ed318070b26861fda2d591a4356fdbeb6c7fdc965be43bddb010fd8299af1286"
+}

backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json

@@ -81,7 +81,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.438.0"
+version = "1.441.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -30,7 +30,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.438.0"
+version = "1.441.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -47,28 +47,40 @@ lto = "thin"
 
 [features]
 default = []
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
-enterprise_saml = ["windmill-api/enterprise_saml"]
-stripe = ["windmill-api/stripe"]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
+enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
+stripe = ["windmill-api/stripe", "enterprise"]
 benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
-flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
 embedding = ["windmill-api/embedding"]
-parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"]
-prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
+parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"]
+prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
 flow_testing = ["windmill-worker/flow_testing"]
 openidconnect = ["windmill-api/openidconnect"]
 cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
-tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
+tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "enterprise", "parquet"]
 sqlx = ["windmill-worker/sqlx"]
 deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
 kafka = ["windmill-api/kafka"]
 otel = ["windmill-common/otel", "windmill-worker/otel"]
 dind = ["windmill-worker/dind"]
 php = ["windmill-worker/php"]
+rust = ["windmill-worker/rust"]
 mysql = ["windmill-worker/mysql"]
+mssql = ["windmill-worker/mssql"]
+bigquery = ["windmill-worker/bigquery"]
+websocket = ["windmill-api/websocket"]
+python = ["windmill-worker/python"]
+smtp = ["windmill-api/smtp", "windmill-common/smtp"]
 csharp = ["windmill-worker/csharp"]
+license = ["windmill-api/license"]
+oauth2 = ["windmill-api/oauth2"]
+http_trigger = ["windmill-api/http_trigger"]
+zip = ["windmill-api/zip"]
+static_frontend = ["windmill-api/static_frontend"]
+scoped_cache = ["windmill-common/scoped_cache"]
+
 
 [dependencies]
 anyhow.workspace = true
@@ -92,7 +104,7 @@ sha2.workspace = true
 url.workspace = true
 lazy_static.workspace = true
 once_cell.workspace = true
-prometheus.workspace = true
+prometheus = { workspace = true, optional = true }
 uuid.workspace = true
 gethostname.workspace = true
 serde_json.workspace = true
@@ -170,7 +182,7 @@ rand = "^0"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
-rustpython-parser = { git = "https://github.com/RustPython/Parser", rev = "9ce55aefdeb35e2f706ce0b02d5a2dfe6295fc57" }
+rustpython-parser = "^0"
 php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
 cron = "^0"
 mail-send = { version = "0.4.0", features = ["builder"], default-features=false }
@@ -229,13 +241,13 @@ lazy_static = "1.4.0"
 serde_derive = "1.0.147"
 const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
 dyn-iter = "0.2.0"
-rsa = "0.7.2"
+rsa = "^0"
 async-stripe = { version = "0.39.1", features = [
     "runtime-tokio-hyper",
     "checkout",
     "billing",
 ] }
-async_zip = { version = "0.0.11", features = ["full"] }
+async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] }
 once_cell = "1.17.1"
 gosyn = "0.2.6"
 bytes = "1.4.0"
@@ -265,7 +277,7 @@ tokenizers = "0.14.1"
 candle-core = "0.3.0"
 candle-transformers = "0.3.0"
 candle-nn = "0.3.0"
-tiberius = { git = "https://github.com/prisma/tiberius", rev = "8f66a699dfa041e7b5f736c7e94f92c945453c9e", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]}
+tiberius = { version = "0.12.3", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]}
 pin-project = "1"
 indexmap = { version = "2.2.5", features = ["serde"]}
 tokio-native-tls = "^0"
@@ -276,8 +288,7 @@ rdkafka = { version = "0.36.2", features = ["cmake-build", "ssl-vendored"] }
 
 datafusion = "39.0.0"
 object_store = { version = "0.10.0", features = ["aws", "azure"] }
-openidconnect = { version = "3.4.0" }
-zstd = "=0.12.4"
+openidconnect = { version = "4.0.0-rc.1" }
 aws-config = "^1"
 aws-sdk-sts = "^1"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-dddc8d60d483a2ce8d78233a25a3899a2b1224ca
+586b02014d57f862a5c4313dd1e529d50c315c30

backend/src/main.rs

@@ -8,7 +8,9 @@
 
 use anyhow::Context;
 use monitor::{
-    load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, reload_nuget_config_setting, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store
+    load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
+    reload_nuget_config_setting, reload_timeout_wait_result_setting,
+    send_current_log_file_to_object_store, send_logs_to_object_store,
 };
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -29,7 +31,16 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE
 
 use windmill_common::{
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
+        DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
+        EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
+        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
+        RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
+        TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -64,7 +75,7 @@ use windmill_worker::{
 
 use crate::monitor::{
     initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
-    load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, monitor_pool,
+    load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db,
     reload_base_url_setting, reload_bunfig_install_scopes_setting,
     reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting,
     reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
@@ -131,7 +142,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
 
     for path in paths.values() {
         tracing::info!("Caching hub script at {path}");
-        let res = get_hub_script_content_and_requirements(Some(path.to_string()), None).await?;
+        let res = get_hub_script_content_and_requirements(Some(path), None).await?;
         if res
             .language
             .as_ref()
@@ -178,7 +189,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
 
                 if let Err(e) = windmill_worker::prebundle_bun_script(
                     &res.content,
-                    Some(lockfile),
+                    Some(&lockfile),
                     &path,
                     &job_id,
                     "admins",
@@ -272,9 +283,6 @@ async fn windmill_main() -> anyhow::Result<()> {
     #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
     println!("jemalloc enabled");
 
-    #[cfg(feature = "flamegraph")]
-    let _guard = windmill_common::tracing_init::setup_flamegraph();
-
     let cli_arg = std::env::args().nth(1).unwrap_or_default();
 
     match cli_arg.as_str() {
@@ -351,7 +359,6 @@ async fn windmill_main() -> anyhow::Result<()> {
         .unwrap_or_else(|| "local")
         .to_string();
 
-    #[cfg(not(feature = "flamegraph"))]
     let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
 
     let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
@@ -478,7 +485,8 @@ Windmill Community Edition {GIT_VERSION}
         )
         .await;
 
-        monitor_pool(&db).await;
+        #[cfg(feature = "prometheus")]
+        crate::monitor::monitor_pool(&db).await;
 
         send_logs_to_object_store(&db, &hostname, &mode);
 
@@ -601,6 +609,7 @@ Windmill Community Edition {GIT_VERSION}
                     server_killpill_rx,
                     base_internal_tx,
                     server_mode,
+                    #[cfg(feature = "smtp")]
                     base_internal_url.clone(),
                 )
                 .await?;

backend/src/monitor.rs

@@ -1,5 +1,6 @@
+#[cfg(feature = "oauth2")]
+use std::collections::HashMap;
 use std::{
-    collections::HashMap,
     fmt::Display,
     ops::Mul,
     str::FromStr,
@@ -22,19 +23,30 @@ use tokio::{
 #[cfg(feature = "embedding")]
 use windmill_api::embeddings::update_embeddings_db;
 use windmill_api::{
-    jobs::TIMEOUT_WAIT_RESULT,
-    oauth2_ee::{build_oauth_clients, OAuthClient},
-    DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
+    jobs::TIMEOUT_WAIT_RESULT, DEFAULT_BODY_LIMIT, IS_SECURE, REQUEST_SIZE_LIMIT, SAML_METADATA,
+    SCIM_TOKEN,
 };
+
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
+
+#[cfg(feature = "oauth2")]
+use windmill_common::global_settings::OAUTH_SETTING;
 use windmill_common::{
     auth::JWT_SECRET,
     ee::CriticalErrorChannel,
     error,
     flow_status::FlowStatusModule,
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
+        DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
+        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
+        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING,
+        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     indexer::load_indexer_config,
     jobs::QueuedJob,
@@ -55,7 +67,9 @@ use windmill_common::{
 };
 use windmill_queue::cancel_job;
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY
+    create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender,
+    SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY,
+    NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
 #[cfg(feature = "parquet")]
@@ -72,22 +86,8 @@ use crate::ee::verify_license_key;
 
 use crate::ee::set_license_key;
 
+#[cfg(feature = "prometheus")]
 lazy_static::lazy_static! {
-    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
-    .ok()
-    .and_then(|x| x.parse::<String>().ok())
-    .unwrap_or_else(|| "60".to_string());
-
-    static ref FLOW_ZOMBIE_TRANSITION_TIMEOUT: String = std::env::var("FLOW_ZOMBIE_TRANSITION_TIMEOUT")
-    .ok()
-    .and_then(|x| x.parse::<String>().ok())
-    .unwrap_or_else(|| "60".to_string());
-
-
-    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
-    .ok()
-    .and_then(|x| x.parse::<bool>().ok())
-    .unwrap_or(true);
 
     static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
         "queue_zombie_restart_count",
@@ -106,6 +106,26 @@ lazy_static::lazy_static! {
         &["tag"]
     ).unwrap();
 
+}
+lazy_static::lazy_static! {
+    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
+    .ok()
+    .and_then(|x| x.parse::<String>().ok())
+    .unwrap_or_else(|| "60".to_string());
+
+    static ref FLOW_ZOMBIE_TRANSITION_TIMEOUT: String = std::env::var("FLOW_ZOMBIE_TRANSITION_TIMEOUT")
+    .ok()
+    .and_then(|x| x.parse::<String>().ok())
+    .unwrap_or_else(|| "60".to_string());
+
+
+    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
+    .ok()
+    .and_then(|x| x.parse::<bool>().ok())
+    .unwrap_or(true);
+
+
+
     static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
 
 }
@@ -324,6 +344,7 @@ pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
 #[derive(Debug, Clone)]
 pub struct MallctlError {
+    #[allow(unused)]
     pub code: i32,
 }
 
@@ -1149,6 +1170,7 @@ pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
     Ok(())
 }
 
+#[cfg(feature = "prometheus")]
 pub async fn monitor_pool(db: &DB) {
     if METRICS_ENABLED.load(Ordering::Relaxed) {
         let db = db.clone();
@@ -1274,6 +1296,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
     if metrics_enabled || save_metrics {
         let queue_counts = windmill_common::queue::get_queue_counts(db).await;
 
+        #[cfg(feature = "prometheus")]
         if metrics_enabled {
             for q in QUEUE_COUNT_TAGS.read().await.iter() {
                 if queue_counts.get(q).is_none() {
@@ -1282,11 +1305,13 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             }
         }
 
+        #[allow(unused_mut)]
         let mut tags_to_watch = vec![];
         for q in queue_counts {
             let count = q.1;
             let tag = q.0;
 
+            #[cfg(feature = "prometheus")]
             if metrics_enabled {
                 let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
                 metric.set(count as i64);
@@ -1430,10 +1455,15 @@ pub async fn load_base_url(db: &DB) -> error::Result<String> {
 }
 
 pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
+    #[cfg(feature = "oauth2")]
     let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
 
+    #[cfg(feature = "oauth2")]
     let oauths = if let Some(q) = q_oauth {
-        if let Ok(v) = serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.clone()) {
+        if let Ok(v) = serde_json::from_value::<
+            Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
+        >(q.clone())
+        {
             v
         } else {
             tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q);
@@ -1446,9 +1476,10 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
     let base_url = load_base_url(db).await?;
     let is_secure = base_url.starts_with("https://");
 
+    #[cfg(feature = "oauth2")]
     {
-        let mut l = OAUTH_CLIENTS.write().await;
-        *l = build_oauth_clients(&base_url, oauths)
+        let mut l = windmill_api::OAUTH_CLIENTS.write().await;
+        *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths)
         .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
         .unwrap();
     }
@@ -1474,9 +1505,11 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
             .ok()
             .unwrap_or_else(|| vec![]);
 
+        #[cfg(feature = "prometheus")]
         if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
             QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
         }
+
         let base_url = BASE_URL.read().await.clone();
         for r in restarted {
             let last_ping = if let Some(x) = r.last_ping {
@@ -1513,6 +1546,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
         .ok()
         .unwrap_or_else(|| vec![]);
 
+    #[cfg(feature = "prometheus")]
     if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
         QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
     }

backend/tests/fixtures/hello.sql

@@ -0,0 +1,54 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export function main(world: string) {
+    const greet = `Hello ${world}!`;
+    console.log(greet)
+    return greet
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/hello', 123412, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
+'test-workspace',
+'',
+'',
+'f/system/hello_flow',
+'{1443253234253453}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
+'{"modules": [{"id": "a", "value": {"path": "f/system/hello", "type": "script", "input_transforms": {"world": {"expr": "flow_input.world", "type": "javascript"}}}}]}',
+'system'
+);
+
+INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES (
+1443253234253453,
+'test-workspace',
+'f/system/hello_flow',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'system'
+);
+
+INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
+'test-workspace',
+'',
+'',
+'f/system/hello_with_nodes_flow',
+'{1443253234253454}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
+E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n    const greet = `Hello ${world}!`;\\n    console.log(greet)\\n    return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n    const dareyou = `Did you just say \\"${hello}\\"??!`;\\n    console.log(dareyou)\\n    return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[\'foo\', \'bar\', \'baz\']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
+'system'
+);
+
+INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES (
+1443253234253454,
+'test-workspace',
+'f/system/hello_with_nodes_flow',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
+E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n    const greet = `Hello ${world}!`;\\n    console.log(greet)\\n    return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n    const dareyou = `Did you just say \\"${hello}\\"??!`;\\n    console.log(dareyou)\\n    return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[\'foo\', \'bar\', \'baz\']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
+'system'
+);

backend/tests/fixtures/relative_bun.sql

@@ -9,7 +9,7 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system/same_folder_script', -28028598712388162, 'bun', '');
+'f/system/same_folder_script', 12340, 'bun', '');
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
 'test-workspace',
@@ -22,7 +22,7 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
+'f/system_relative/different_folder_script', 12341, 'bun', '');
 
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
@@ -41,4 +41,4 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/nested_script', -28028598712388160, 'bun', '');
+'f/system_relative/nested_script', 12342, 'bun', '');

backend/tests/fixtures/relative_deno.sql

@@ -9,7 +9,7 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system/same_folder_script', -28028598712388162, 'deno', '');
+'f/system/same_folder_script', 12343, 'deno', '');
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
 'test-workspace',
@@ -22,7 +22,7 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
+'f/system_relative/different_folder_script', 12344, 'deno', '');
 
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
@@ -41,4 +41,4 @@ export function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/nested_script', -28028598712388160, 'deno', '');
+'f/system_relative/nested_script', 12345, 'deno', '');

backend/tests/fixtures/relative_python.sql

@@ -8,7 +8,7 @@ def main():
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system/same_folder_script', -28028598712388162, 'python3', '');
+'f/system/same_folder_script', 12346, 'python3', '');
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
 'test-workspace',
@@ -20,7 +20,7 @@ def main():
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
+'f/system_relative/different_folder_script', 12347, 'python3', '');
 
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
@@ -38,4 +38,4 @@ def main():
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
 '',
 '',
-'f/system_relative/nested_script', -28028598712388160, 'python3', '');
+'f/system_relative/nested_script', 12348, 'python3', '');

backend/tests/fixtures/schedule.sql

@@ -13,7 +13,7 @@ export async function main(fail: boolean = true) {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
 '',
 '',
-'f/system/failing_script', -28028598712388162, 'deno', '');
+'f/system/failing_script', 12349, 'deno', '');
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
 'test-workspace',
@@ -26,7 +26,7 @@ export async function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
 '',
 '',
-'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+'f/system/schedule_error_handler', 123410, 'deno', '');
 
 INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
 'test-workspace',
@@ -39,21 +39,21 @@ export async function main() {
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
 '',
 '',
-'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+'f/system/schedule_recovery_handler', 123411, 'deno', '');
 
 INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
 'test-workspace',
 '',
 '',
 'f/system/failing_flow',
-'{1}',
+'{1443253234253452}',
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
 '{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
 'system'
 );
 
 INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) VALUES (
-1,
+1443253234253452,
 'test-workspace',
 'f/system/failing_flow',
 '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',

backend/tests/worker.rs

@@ -1,5 +1,6 @@
 use serde::de::DeserializeOwned;
-use std::str::FromStr;
+use std::future::Future;
+use std::{str::FromStr, sync::Arc};
 use windmill_api_client::types::{NewScript, NewScriptLanguage};
 
 #[cfg(feature = "enterprise")]
@@ -11,6 +12,7 @@ use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
 
+use tokio::sync::RwLock;
 #[cfg(feature = "enterprise")]
 use tokio::time::{timeout, Duration};
 
@@ -29,6 +31,9 @@ use windmill_common::{
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
     jobs::{JobKind, JobPayload, RawCode},
     scripts::{ScriptHash, ScriptLang},
+    worker::{
+        MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440,
+    },
 };
 use windmill_queue::PushIsolationLevel;
 
@@ -134,12 +139,16 @@ impl ApiServer {
             rx,
             port_tx,
             false,
+            #[cfg(feature = "smtp")]
             format!("http://localhost:{}", addr.port()),
         ));
 
         _port_rx.await.unwrap();
 
-        return Self { addr, tx, task };
+        // clear the cache between tests
+        windmill_common::cache::clear();
+
+        Self { addr, tx, task }
     }
 
     async fn close(self) -> anyhow::Result<()> {
@@ -3731,3 +3740,472 @@ async fn test_result_format(db: Pool<Postgres>) {
     .unwrap();
     assert_eq!(result.get(), correct_result);
 }
+
+async fn test_for_versions<F: Future<Output = ()>>(
+    version_flags: impl Iterator<Item = Arc<RwLock<bool>>>,
+    test: impl Fn() -> F,
+) {
+    for version_flag in version_flags {
+        *version_flag.write().await = true;
+        test().await;
+    }
+}
+
+mod job_payload {
+    use super::*;
+
+    use lazy_static::lazy_static;
+
+    use windmill_common::cache;
+    use windmill_common::flows::FlowNodeId;
+
+    lazy_static! {
+        static ref VERSION_FLAGS: [Arc<RwLock<bool>>; 3] = [
+            MIN_VERSION_IS_AT_LEAST_1_427.clone(),
+            MIN_VERSION_IS_AT_LEAST_1_432.clone(),
+            MIN_VERSION_IS_AT_LEAST_1_440.clone(),
+        ];
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_script_hash_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::ScriptHash {
+                hash: ScriptHash(123412),
+                path: "f/system/hello".to_string(),
+                custom_concurrency_key: None,
+                concurrent_limit: None,
+                concurrency_time_window_s: None,
+                cache_ttl: None,
+                dedicated_worker: None,
+                language: ScriptLang::Deno,
+                priority: None,
+                apply_preprocessor: false,
+            })
+            .arg("world", json!("foo"))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(result, json!("Hello foo!"));
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_flow_script_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        // Deploy the flow to produce the "lite" version.
+        let _ = RunJob::from(JobPayload::FlowDependencies {
+            path: "f/system/hello_with_nodes_flow".to_string(),
+            dedicated_worker: None,
+            version: 1443253234253454,
+        })
+        .run_until_complete(&db, port)
+        .await
+        .json_result()
+        .unwrap();
+
+        let flow_data = cache::flow::fetch_version_lite(&db, 1443253234253454)
+            .await
+            .unwrap();
+        let flow_value = flow_data.value();
+        let flow_scripts = {
+            async fn load(db: &Pool<Postgres>, modules: &[FlowModule]) -> Vec<FlowNodeId> {
+                let mut res = vec![];
+                for module in modules {
+                    let value =
+                        serde_json::from_str::<FlowModuleValue>(module.value.get()).unwrap();
+                    match value {
+                        FlowModuleValue::FlowScript { id, .. } => res.push(id),
+                        FlowModuleValue::ForloopFlow { modules_node: Some(flow_node), .. } => {
+                            let flow_data = cache::flow::fetch_flow(db, flow_node).await.unwrap();
+                            res.extend(Box::pin(load(db, &flow_data.value().modules)).await);
+                        }
+                        _ => {}
+                    }
+                }
+                res
+            }
+
+            load(&db, &flow_value.modules).await
+        };
+        assert_eq!(flow_scripts.len(), 2);
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::FlowScript {
+                id: flow_scripts[0],
+                language: ScriptLang::Deno,
+                custom_concurrency_key: None,
+                concurrent_limit: None,
+                concurrency_time_window_s: None,
+                cache_ttl: None,
+                dedicated_worker: None,
+                path: "f/system/hello/test-0".into(),
+            })
+            .arg("world", json!("foo"))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(result, json!("Hello foo!"));
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        let test = || async {
+            let result = RunJob::from(JobPayload::FlowScript {
+                id: flow_scripts[1],
+                language: ScriptLang::Deno,
+                custom_concurrency_key: None,
+                concurrent_limit: None,
+                concurrency_time_window_s: None,
+                cache_ttl: None,
+                dedicated_worker: None,
+                path: "f/system/hello/test-0".into(),
+            })
+            .arg("hello", json!("You know nothing Jean Neige"))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result,
+                json!("Did you just say \"You know nothing Jean Neige\"??!")
+            );
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_flow_node_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        // Deploy the flow to produce the "lite" version.
+        let _ = RunJob::from(JobPayload::FlowDependencies {
+            path: "f/system/hello_with_nodes_flow".to_string(),
+            dedicated_worker: None,
+            version: 1443253234253454,
+        })
+        .run_until_complete(&db, port)
+        .await
+        .json_result()
+        .unwrap();
+
+        let flow_data = cache::flow::fetch_version_lite(&db, 1443253234253454)
+            .await
+            .unwrap();
+        let flow_value = flow_data.value();
+        let forloop_module =
+            serde_json::from_str::<FlowModuleValue>(flow_value.modules[0].value.get()).unwrap();
+        let FlowModuleValue::ForloopFlow { modules_node: Some(id), .. } = forloop_module else {
+            panic!("Expected a forloop module with a flow node");
+        };
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::FlowNode {
+                id,
+                path: "f/system/hello_with_nodes_flow/forloop-0".into(),
+            })
+            .arg("iter", json!({ "value": "tests", "index": 0 }))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(result, json!("Did you just say \"Hello tests!\"??!"));
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_dependencies_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::Dependencies {
+                path: "f/system/hello".to_string(),
+                hash: ScriptHash(123412),
+                language: ScriptLang::Deno,
+                dedicated_worker: None,
+            })
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result.get("status").unwrap(),
+                &json!("Successful lock file generation")
+            );
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    // Just test that deploying a flow work as expected.
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_flow_dependencies_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::FlowDependencies {
+                path: "f/system/hello_with_nodes_flow".to_string(),
+                dedicated_worker: None,
+                version: 1443253234253454,
+            })
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result.get("status").unwrap(),
+                &json!("Successful lock file generation")
+            );
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_raw_flow_dependencies_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::RawFlowDependencies {
+                path: "none".to_string(),
+                flow_value: serde_json::from_value(json!({
+                    "modules": [{
+                        "id": "a",
+                        "value": {
+                            "type": "rawscript",
+                            "content": r#"export function main(world: string) {
+                                const greet = `Hello ${world}!`;
+                                console.log(greet)
+                                return greet
+                            }"#,
+                            "language": "deno",
+                            "input_transforms": {
+                                "world": { "type": "javascript", "expr": "flow_input.world" }
+                            }
+                        }
+                    }],
+                    "schema": {
+                        "$schema": "https://json-schema.org/draft/2020-12/schema",
+                        "properties": { "world": { "type": "string" } },
+                        "type": "object",
+                        "order": [  "world" ]
+                    }
+                }))
+                .unwrap(),
+            })
+            .arg("skip_flow_update", json!(true))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            let result = RunJob::from(JobPayload::RawFlow {
+                value: serde_json::from_value::<FlowValue>(
+                    result.get("updated_flow_value").unwrap().clone(),
+                )
+                .unwrap(),
+                path: None,
+                restarted_from: None,
+            })
+            .arg("world", json!("Jean Neige"))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(result, json!("Hello Jean Neige!"));
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_raw_script_dependencies_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::RawScriptDependencies {
+                script_path: "none".into(),
+                content: r#"export function main(world: string) {
+                    const greet = `Hello ${world}!`;
+                    console.log(greet)
+                    return greet
+                }"#
+                .into(),
+                language: ScriptLang::Deno,
+            })
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result,
+                json!({ "lock": "", "status": "Successful lock file generation" })
+            );
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_flow_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::Flow {
+                path: "f/system/hello_with_nodes_flow".to_string(),
+                dedicated_worker: None,
+                apply_preprocessor: true,
+            })
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result,
+                json!([
+                    "Did you just say \"Hello foo!\"??!",
+                    "Did you just say \"Hello bar!\"??!",
+                    "Did you just say \"Hello baz!\"??!",
+                ])
+            );
+        };
+        // Test the not "lite" flow.
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        // Deploy the flow to produce the "lite" version.
+        let _ = RunJob::from(JobPayload::FlowDependencies {
+            path: "f/system/hello_with_nodes_flow".to_string(),
+            dedicated_worker: None,
+            version: 1443253234253454,
+        })
+        .run_until_complete(&db, port)
+        .await
+        .json_result()
+        .unwrap();
+        // Test the "lite" flow.
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_restarted_flow_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let completed_job_id = RunJob::from(JobPayload::Flow {
+                path: "f/system/hello_with_nodes_flow".to_string(),
+                dedicated_worker: None,
+                apply_preprocessor: true,
+            })
+            .run_until_complete(&db, port)
+            .await
+            .id;
+
+            let result = RunJob::from(JobPayload::RestartedFlow {
+                completed_job_id,
+                step_id: "a".into(),
+                branch_or_iteration_n: None,
+            })
+            .arg("iter", json!({ "value": "tests", "index": 0 }))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(
+                result,
+                json!([
+                    "Did you just say \"Hello foo!\"??!",
+                    "Did you just say \"Hello bar!\"??!",
+                    "Did you just say \"Hello baz!\"??!",
+                ])
+            );
+        };
+        // Test the not "lite" flow.
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+        // Deploy the flow to produce the "lite" version.
+        let _ = RunJob::from(JobPayload::FlowDependencies {
+            path: "f/system/hello_with_nodes_flow".to_string(),
+            dedicated_worker: None,
+            version: 1443253234253454,
+        })
+        .run_until_complete(&db, port)
+        .await
+        .json_result()
+        .unwrap();
+        // Test the "lite" flow.
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+
+    #[sqlx::test(fixtures("base", "hello"))]
+    async fn test_raw_flow_payload(db: Pool<Postgres>) {
+        initialize_tracing().await;
+        let server = ApiServer::start(db.clone()).await;
+        let port = server.addr.port();
+
+        let test = || async {
+            let result = RunJob::from(JobPayload::RawFlow {
+                value: serde_json::from_value(json!({
+                    "modules": [{
+                        "id": "a",
+                        "value": {
+                            "type": "rawscript",
+                            "content": r#"export function main(world: string) {
+                                const greet = `Hello ${world}!`;
+                                console.log(greet)
+                                return greet
+                            }"#,
+                            "language": "deno",
+                            "input_transforms": {
+                                "world": { "type": "javascript", "expr": "flow_input.world" }
+                            }
+                        }
+                    }],
+                    "schema": {
+                        "$schema": "https://json-schema.org/draft/2020-12/schema",
+                        "properties": { "world": { "type": "string" } },
+                        "type": "object",
+                        "order": [  "world" ]
+                    }
+                }))
+                .unwrap(),
+                path: None,
+                restarted_from: None,
+            })
+            .arg("world", json!("Jean Neige"))
+            .run_until_complete(&db, port)
+            .await
+            .json_result()
+            .unwrap();
+
+            assert_eq!(result, json!("Hello Jean Neige!"));
+        };
+        test_for_versions(VERSION_FLAGS.iter().cloned(), test).await;
+    }
+}

backend/windmill-api/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/lib.rs"
 
 [features]
 default = []
-enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:openidconnect"]
+enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"]
 stripe = ["dep:async-stripe"]
 enterprise_saml = ["dep:samael"]
 benchmark = []
@@ -20,13 +20,19 @@ prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:pr
 openidconnect = ["dep:openidconnect"]
 tantivy = ["dep:windmill-indexer"]
 kafka = ["dep:rdkafka"]
+websocket = ["dep:tokio-tungstenite"]
+smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"]
+license = ["dep:rsa"]
+zip = ["dep:async_zip"]
+oauth2 = ["dep:async-oauth2"]
+http_trigger = ["dep:matchit"]
+static_frontend = ["dep:rust-embed"]
 
 [dependencies]
 windmill-queue.workspace = true
 windmill-common = { workspace = true, default-features = false }
 windmill-audit.workspace = true
 windmill-parser.workspace = true
-windmill-parser-py-imports.workspace = true
 windmill-parser-ts.workspace = true
 windmill-git-sync.workspace = true
 windmill-indexer = { workspace = true, optional = true }
@@ -44,7 +50,7 @@ itertools.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 sqlx.workspace = true
-async-oauth2.workspace = true
+async-oauth2 = { workspace = true, optional = true }
 tracing.workspace = true
 sql-builder.workspace = true
 serde_json.workspace = true
@@ -56,15 +62,15 @@ base32.workspace = true
 serde_urlencoded.workspace = true
 cron.workspace = true
 mime_guess.workspace = true
-rust-embed.workspace = true
+rust-embed = { workspace = true, optional = true }
 tracing-subscriber.workspace = true
 quick_cache.workspace = true
 rand.workspace = true
 time.workspace = true
 native-tls.workspace = true
 tokio-native-tls.workspace = true
-openssl.workspace = true
-mail-parser = { workspace = true, features = ["serde_support"] }
+openssl = { workspace = true, optional = true }
+mail-parser = { workspace = true, features = ["serde_support"], optional = true }
 magic-crypt.workspace = true
 tempfile.workspace = true
 tokio-util.workspace = true
@@ -76,12 +82,12 @@ urlencoding.workspace = true
 async-stripe = { workspace = true, optional = true }
 lazy_static.workspace = true
 prometheus = { workspace = true, optional = true }
-async_zip.workspace = true
+async_zip = { workspace = true, optional = true }
 regex.workspace = true
 bytes.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
-rsa.workspace = true
+rsa = { workspace = true, optional = true}
 uuid.workspace = true
 tinyvector = { workspace = true, optional = true}
 hf-hub  = { workspace = true, optional = true}
@@ -94,8 +100,8 @@ object_store = { workspace = true, optional = true}
 openidconnect = { workspace = true, optional = true}
 url = { workspace = true, optional = true}
 jsonwebtoken = { workspace = true }
-matchit.workspace = true
-tokio-tungstenite.workspace = true
+matchit = { workspace = true, optional = true }
+tokio-tungstenite = { workspace = true, optional = true}
 rdkafka = { workspace = true, optional = true }
 const_format.workspace = true
 

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.438.0
+  version: 1.441.0
   title: Windmill API
 
   contact:
@@ -1121,9 +1121,15 @@ paths:
       operationId: refreshUserToken
       tags:
         - user
+      parameters:
+        - name: if_expiring_in_less_than_s
+          in: query
+          required: false
+          schema:
+            type: integer
       responses:
         "200":
-          description: free usage
+          description: new token
           content:
             text/plain:
               schema:
@@ -6937,6 +6943,42 @@ paths:
                   - resume
                   - cancel
 
+  /w/{workspace}/jobs/slack_approval/{id}:
+    get:
+      summary: generate interactive slack approval for suspended job
+      operationId: getSlackApprovalPayload
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+        - name: approver
+          in: query
+          schema:
+            type: string
+        - name: message
+          in: query
+          schema:
+            type: string
+        - name: slack_resource_path
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: channel_id
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: flow_step_id
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Interactive slack approval message sent successfully
+
   /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}:
     get:
       summary: resume a job for a suspended flow

backend/windmill-api/src/apps.rs

@@ -20,7 +20,7 @@ use crate::{
 #[cfg(feature = "parquet")]
 use crate::{
     job_helpers_ee::{
-        get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_internal,
+        get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_from_req,
         UploadFileResponse,
     },
     users::fetch_api_authed_from_permissioned_as,
@@ -101,11 +101,6 @@ pub fn global_service() -> Router {
         .route("/hub/get/:id", get(get_hub_app_by_id))
 }
 
-#[cfg(not(feature = "enterprise"))]
-pub fn global_unauthed_service() -> Router {
-    Router::new()
-}
-
 #[derive(FromRow, Deserialize, Serialize)]
 pub struct ListableApp {
     pub id: i64,
@@ -1327,10 +1322,7 @@ async fn execute_component(
             // 2. Otherwise, always fetch the policy from the database.
             let policy = if let Some(id) = payload.version {
                 let cache = cache::anon!({ u64 => Arc<Policy> } in "policy" <= 1000);
-                arc_policy = policy_fut
-                    .map_ok(Arc::new)
-                    .cached(cache, &(id as u64))
-                    .await?;
+                arc_policy = policy_fut.map_ok(Arc::new).cached(cache, id as u64).await?;
                 &*arc_policy
             } else {
                 policy = policy_fut.await?;
@@ -1358,7 +1350,7 @@ async fn execute_component(
                     .fetch_one(&db)
                     .map_err(Into::<Error>::into)
                     .map_ok(Arc::new)
-                    .cached(cache, &(*id as u64))
+                    .cached(cache, *id as u64)
                     .await?
                 }
                 _ => unreachable!(),
@@ -1681,7 +1673,7 @@ async fn upload_s3_file_from_app(
     ])
     .into();
 
-    upload_file_internal(s3_client, &file_key, request, options).await?;
+    upload_file_from_req(s3_client, &file_key, request, options).await?;
 
     return Ok(Json(UploadFileResponse { file_key }));
 }

backend/windmill-api/src/args.rs

@@ -3,12 +3,12 @@ use std::collections::HashMap;
 #[cfg(feature = "parquet")]
 use crate::job_helpers_ee::get_workspace_s3_resource;
 use axum::{
-    extract::{FromRequest, FromRequestParts, Query, Request},
+    extract::{FromRequest, FromRequestParts, Multipart, Query, Request},
     http::{HeaderMap, Uri},
     response::{IntoResponse, Response},
 };
 use bytes::Bytes;
-use http::{header::CONTENT_TYPE, request::Parts};
+use http::{header::CONTENT_TYPE, request::Parts, StatusCode};
 #[cfg(feature = "parquet")]
 use object_store::{Attribute, Attributes};
 use serde::Deserialize;
@@ -23,9 +23,11 @@ use crate::db::ApiAuthed;
 #[cfg(feature = "parquet")]
 use crate::job_helpers_ee::{get_random_file_name, upload_file_internal};
 
+#[derive(Default)]
 pub struct WebhookArgs {
     pub args: PushArgsOwned,
-    pub file_req: Option<Request>,
+    pub multipart: Option<Multipart>,
+    pub wrap_body: Option<bool>,
 }
 
 impl WebhookArgs {
@@ -36,9 +38,9 @@ impl WebhookArgs {
         _db: &DB,
         _w_id: &str,
     ) -> Result<PushArgsOwned, Error> {
-        if self.file_req.is_some() {
+        if self.multipart.is_some() {
             return Err(Error::BadRequest(format!(
-                "Uploading files requires the parquet feature"
+                "multipart/form-data requires the parquet feature"
             )));
         }
 
@@ -52,7 +54,9 @@ impl WebhookArgs {
         db: &DB,
         w_id: &str,
     ) -> Result<PushArgsOwned, Error> {
-        if let Some(req) = self.file_req {
+        use futures::TryStreamExt;
+
+        if let Some(mut multipart) = self.multipart {
             {
                 let (_, s3_resource) =
                     get_workspace_s3_resource(authed, db, None, "", w_id, None).await?;
@@ -60,47 +64,78 @@ impl WebhookArgs {
                 if let Some(s3_resource) = s3_resource {
                     let s3_client = build_object_store_client(&s3_resource).await?;
 
-                    let content_type = req
-                        .headers()
-                        .get(CONTENT_TYPE)
-                        .map(|x| x.to_str().ok().map(|x| x.to_string()))
-                        .flatten();
+                    let mut body = HashMap::new();
 
-                    let file_extension = content_type
-                        .as_ref()
-                        .map(|mime_str| {
-                            mime_guess::get_mime_extensions_str(mime_str)
-                                .map(|x| x.first().map(|x| x.to_string()))
-                        })
-                        .flatten()
-                        .flatten();
+                    while let Some(field) = multipart.next_field().await.map_err(|e| {
+                        Error::BadRequest(format!(
+                            "Error reading multipart field: {}",
+                            e.body_text()
+                        ))
+                    })? {
+                        if let Some(name) = field.name().map(|x| x.to_string()) {
+                            if let Some(content_type) = field.content_type() {
+                                let ext = field
+                                    .file_name()
+                                    .map(|x| x.split('.').last())
+                                    .flatten()
+                                    .map(|x| x.to_string());
 
-                    let file_key = get_random_file_name(file_extension);
+                                let file_key = get_random_file_name(ext);
 
-                    let options = Attributes::from_iter(vec![
-                        (
-                            Attribute::ContentType,
-                            content_type.unwrap_or("application/octet-stream".to_string()),
-                        ),
-                        (Attribute::ContentDisposition, "inline".to_string()),
-                    ])
-                    .into();
+                                let options = Attributes::from_iter(vec![
+                                    (Attribute::ContentType, content_type.to_string()),
+                                    (
+                                        Attribute::ContentDisposition,
+                                        if let Some(filename) = field.file_name() {
+                                            format!("inline; filename=\"{}\"", filename)
+                                        } else {
+                                            "inline".to_string()
+                                        },
+                                    ),
+                                ])
+                                .into();
 
-                    upload_file_internal(s3_client, &file_key, req, options).await?;
+                                let bytes_stream = field.into_stream().map_err(|err| {
+                                    std::io::Error::new(std::io::ErrorKind::Other, err)
+                                });
 
-                    self.args.args.insert(
-                        "body".to_string(),
-                        to_raw_value(&serde_json::json!({
-                            "s3": &file_key
-                        })),
-                    );
+                                upload_file_internal(
+                                    s3_client.clone(),
+                                    &file_key,
+                                    bytes_stream,
+                                    options,
+                                )
+                                .await?;
+
+                                body.insert(
+                                    name,
+                                    to_raw_value(&serde_json::json!({
+                                        "s3": &file_key
+                                    })),
+                                );
+                            } else {
+                                body.insert(
+                                    name,
+                                    to_raw_value(&field.text().await.unwrap_or_default()),
+                                );
+                            }
+                        }
+                    }
+
+                    if self.wrap_body.unwrap_or(false) {
+                        self.args
+                            .args
+                            .insert("body".to_string(), to_raw_value(&body));
+                    } else {
+                        self.args.args.extend(body);
+                    }
 
                     return Ok(self.args);
                 }
             }
 
             return Err(Error::BadRequest(format!(
-                "You need to connect your workspace to an S3 bucket to upload files"
+                "You need to connect your workspace to an S3 bucket to use multipart/form-data"
             )));
         }
 
@@ -168,7 +203,7 @@ where
                 }
                 return Ok(Self {
                     args: PushArgsOwned { extra: Some(extra), args: args },
-                    file_req: None,
+                    ..Default::default()
                 });
             }
             let str = String::from_utf8(bytes.to_vec())
@@ -176,7 +211,7 @@ where
 
             PushArgsOwned::from_json(extra, use_raw, wrap_body, str)
                 .await
-                .map(|args| Self { args, file_req: None })
+                .map(|args| Self { args, ..Default::default() })
         } else if content_type
             .unwrap()
             .starts_with("application/cloudevents+json")
@@ -185,7 +220,7 @@ where
 
             PushArgsOwned::from_ce_json(extra, use_raw, str)
                 .await
-                .map(|args| Self { args, file_req: None })
+                .map(|args| Self { args, ..Default::default() })
         } else if content_type
             .unwrap()
             .starts_with("application/cloudevents-batch+json")
@@ -199,7 +234,7 @@ where
             extra.insert("raw_string".to_string(), to_raw_value(&str));
             Ok(Self {
                 args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
-                file_req: None,
+                ..Default::default()
             })
         } else if content_type
             .unwrap()
@@ -227,7 +262,7 @@ where
 
             return Ok(Self {
                 args: PushArgsOwned { extra: Some(extra), args: payload },
-                file_req: None,
+                ..Default::default()
             });
         } else if content_type.unwrap().starts_with("application/xml")
             || content_type.unwrap().starts_with("text/xml")
@@ -236,15 +271,20 @@ where
             extra.insert("raw_string".to_string(), to_raw_value(&str));
             Ok(Self {
                 args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
-                file_req: None,
+                ..Default::default()
+            })
+        } else if content_type.unwrap().starts_with("multipart/form-data") {
+            let multipart = Multipart::from_request(req, _state)
+                .await
+                .map_err(IntoResponse::into_response)?;
+
+            Ok(Self {
+                args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
+                multipart: Some(multipart),
+                wrap_body: Some(wrap_body),
             })
         } else {
-            return Ok(Self {
-                args: PushArgsOwned { extra: None, args: HashMap::new() },
-                file_req: Some(req),
-            });
-
-            // Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
+            Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
         }
     }
 }

backend/windmill-api/src/auth.rs

@@ -0,0 +1,587 @@
+#[cfg(feature = "enterprise")]
+use crate::ee::ExternalJwks;
+use axum::{
+    async_trait,
+    extract::{FromRequestParts, OriginalUri, Query},
+    Extension,
+};
+use chrono::TimeZone;
+use http::{request::Parts, StatusCode};
+use quick_cache::sync::Cache;
+use serde::Deserialize;
+use tower_cookies::Cookies;
+use tracing::Span;
+
+use crate::db::{ApiAuthed, DB};
+use std::sync::{
+    atomic::{AtomicI64, AtomicU64, Ordering},
+    Arc,
+};
+#[cfg(feature = "enterprise")]
+use tokio::sync::RwLock;
+
+use windmill_common::{
+    auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET},
+    users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL},
+};
+
+#[derive(Clone)]
+pub struct ExpiringAuthCache {
+    pub authed: ApiAuthed,
+    pub expiry: chrono::DateTime<chrono::Utc>,
+}
+
+pub struct AuthCache {
+    cache: Cache<(String, String), ExpiringAuthCache>,
+    db: DB,
+    superadmin_secret: Option<String>,
+    #[cfg(feature = "enterprise")]
+    ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
+}
+
+impl AuthCache {
+    pub fn new(
+        db: DB,
+        superadmin_secret: Option<String>,
+        #[cfg(feature = "enterprise")] ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
+    ) -> Self {
+        AuthCache {
+            cache: Cache::new(300),
+            db,
+            superadmin_secret,
+            #[cfg(feature = "enterprise")]
+            ext_jwks,
+        }
+    }
+
+    pub async fn invalidate(&self, w_id: &str, token: String) {
+        self.cache.remove(&(w_id.to_string(), token));
+    }
+
+    pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
+        let key = (
+            w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
+            token.to_string(),
+        );
+        let s = self.cache.get(&key).map(|c| c.to_owned());
+        match s {
+            Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => {
+                Some(authed)
+            }
+            #[cfg(feature = "enterprise")]
+            _ if token.starts_with("jwt_ext_") => {
+                let authed_and_exp = match crate::ee::jwt_ext_auth(
+                    w_id.as_ref(),
+                    token.trim_start_matches("jwt_ext_"),
+                    self.ext_jwks.clone(),
+                )
+                .await
+                {
+                    Ok(r) => Some(r),
+                    Err(e) => {
+                        tracing::error!("JWT_EXT auth error: {:?}", e);
+                        None
+                    }
+                };
+
+                if let Some((authed, exp)) = authed_and_exp.clone() {
+                    self.cache.insert(
+                        key,
+                        ExpiringAuthCache {
+                            authed: authed.clone(),
+                            expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000),
+                        },
+                    );
+
+                    Some(authed)
+                } else {
+                    None
+                }
+            }
+            _ if token.starts_with("jwt_") => {
+                let jwt_secret = JWT_SECRET.read().await;
+                if !jwt_secret.is_empty() {
+                    let jwt_token = token.trim_start_matches("jwt_");
+
+                    let jwt_result = jsonwebtoken::decode::<JWTAuthClaims>(
+                        jwt_token,
+                        &jsonwebtoken::DecodingKey::from_secret(jwt_secret.as_bytes()),
+                        &jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256),
+                    );
+
+                    match jwt_result {
+                        Ok(payload) => {
+                            if w_id.is_some_and(|w_id| w_id != payload.claims.workspace_id) {
+                                tracing::error!("JWT auth error: workspace_id mismatch");
+                                return None;
+                            }
+
+                            let username_override =
+                                username_override_from_label(payload.claims.label);
+                            let authed = crate::db::ApiAuthed {
+                                email: payload.claims.email,
+                                username: payload.claims.username,
+                                is_admin: payload.claims.is_admin,
+                                is_operator: payload.claims.is_operator,
+                                groups: payload.claims.groups,
+                                folders: payload.claims.folders,
+                                scopes: None,
+                                username_override,
+                            };
+
+                            self.cache.insert(
+                                key,
+                                ExpiringAuthCache {
+                                    authed: authed.clone(),
+                                    expiry: chrono::Utc
+                                        .timestamp_nanos(payload.claims.exp as i64 * 1_000_000_000),
+                                },
+                            );
+
+                            Some(authed)
+                        }
+                        Err(err) => {
+                            tracing::error!("JWT auth error: {:?}", err);
+                            None
+                        }
+                    }
+                } else {
+                    tracing::error!("JWT auth error: no jwt secret set");
+                    None
+                }
+            }
+            _ => {
+                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>, Option<String>)>(
+                    "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
+                     OR expiration IS NULL) AND (workspace_id IS NULL OR workspace_id = $2) RETURNING owner, email, super_admin, scopes, label",
+                )
+                .bind(token)
+                .bind(w_id.as_ref())
+                .fetch_optional(&self.db)
+                .await
+                .ok()
+                .flatten();
+
+                if let Some(user) = user_o {
+                    let authed_o = {
+                        match user {
+                            (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
+                                let username_override = username_override_from_label(label);
+                                if let Some((prefix, name)) = owner.split_once('/') {
+                                    if prefix == "u" {
+                                        let (is_admin, is_operator) = if super_admin {
+                                            (true, false)
+                                        } else {
+                                            let r = sqlx::query!(
+                                                "SELECT is_admin, operator FROM usr where username = $1 AND \
+                                                 workspace_id = $2 AND disabled = false",
+                                                name,
+                                                &w_id.as_ref().unwrap()
+                                            )
+                                            .fetch_one(&self.db)
+                                            .await
+                                            .ok();
+                                            if let Some(r) = r {
+                                                (r.is_admin, r.operator)
+                                            } else {
+                                                (false, true)
+                                            }
+                                        };
+
+                                        let w_id = &w_id.unwrap();
+                                        let groups =
+                                            get_groups_for_user(w_id, &name, &email, &self.db)
+                                                .await
+                                                .ok()
+                                                .unwrap_or_default();
+
+                                        let folders =
+                                            get_folders_for_user(w_id, &name, &groups, &self.db)
+                                                .await
+                                                .ok()
+                                                .unwrap_or_default();
+
+                                        Some(ApiAuthed {
+                                            email: email,
+                                            username: name.to_string(),
+                                            is_admin,
+                                            is_operator,
+                                            groups,
+                                            folders,
+                                            scopes: None,
+                                            username_override,
+                                        })
+                                    } else {
+                                        let groups = vec![name.to_string()];
+                                        let folders = get_folders_for_user(
+                                            &w_id.unwrap(),
+                                            "",
+                                            &groups,
+                                            &self.db,
+                                        )
+                                        .await
+                                        .ok()
+                                        .unwrap_or_default();
+                                        Some(ApiAuthed {
+                                            email: email,
+                                            username: format!("group-{name}"),
+                                            is_admin: false,
+                                            groups,
+                                            is_operator: false,
+                                            folders,
+                                            scopes: None,
+                                            username_override,
+                                        })
+                                    }
+                                } else {
+                                    let groups = vec![];
+                                    let folders = vec![];
+                                    Some(ApiAuthed {
+                                        email: email,
+                                        username: owner,
+                                        is_admin: super_admin,
+                                        is_operator: true,
+                                        groups,
+                                        folders,
+                                        scopes: None,
+                                        username_override,
+                                    })
+                                }
+                            }
+                            (_, Some(email), super_admin, scopes, label) => {
+                                let username_override = username_override_from_label(label);
+                                if w_id.is_some() {
+                                    let row_o = sqlx::query_as::<_, (String, bool, bool)>(
+                                        "SELECT username, is_admin, operator FROM usr where email = $1 AND \
+                                         workspace_id = $2 AND disabled = false",
+                                    )
+                                    .bind(&email)
+                                    .bind(&w_id.as_ref().unwrap())
+                                    .fetch_optional(&self.db)
+                                    .await
+                                    .unwrap_or(Some(("error".to_string(), false, false)));
+
+                                    match row_o {
+                                        Some((username, is_admin, is_operator)) => {
+                                            let groups = get_groups_for_user(
+                                                &w_id.as_ref().unwrap(),
+                                                &username,
+                                                &email,
+                                                &self.db,
+                                            )
+                                            .await
+                                            .ok()
+                                            .unwrap_or_default();
+
+                                            let folders = get_folders_for_user(
+                                                &w_id.unwrap(),
+                                                &username,
+                                                &groups,
+                                                &self.db,
+                                            )
+                                            .await
+                                            .ok()
+                                            .unwrap_or_default();
+                                            Some(ApiAuthed {
+                                                email,
+                                                username,
+                                                is_admin: is_admin || super_admin,
+                                                is_operator,
+                                                groups,
+                                                folders,
+                                                scopes,
+                                                username_override,
+                                            })
+                                        }
+                                        None if super_admin => Some(ApiAuthed {
+                                            email: email.clone(),
+                                            username: email,
+                                            is_admin: super_admin,
+                                            is_operator: false,
+                                            groups: vec![],
+                                            folders: vec![],
+                                            scopes,
+                                            username_override,
+                                        }),
+                                        None => None,
+                                    }
+                                } else {
+                                    Some(ApiAuthed {
+                                        email: email.to_string(),
+                                        username: email,
+                                        is_admin: super_admin,
+                                        is_operator: true,
+                                        groups: Vec::new(),
+                                        folders: Vec::new(),
+                                        scopes,
+                                        username_override,
+                                    })
+                                }
+                            }
+                            _ => None,
+                        }
+                    };
+                    if let Some(authed) = authed_o.as_ref() {
+                        self.cache.insert(
+                            key,
+                            ExpiringAuthCache {
+                                authed: authed.clone(),
+                                expiry: chrono::Utc::now()
+                                    + chrono::Duration::try_seconds(120).unwrap(),
+                            },
+                        );
+                    }
+                    authed_o
+                } else if self
+                    .superadmin_secret
+                    .as_ref()
+                    .map(|x| x == token)
+                    .unwrap_or(false)
+                {
+                    Some(ApiAuthed {
+                        email: SUPERADMIN_SECRET_EMAIL.to_string(),
+                        username: "superadmin_secret".to_string(),
+                        is_admin: true,
+                        is_operator: false,
+                        groups: Vec::new(),
+                        folders: Vec::new(),
+                        scopes: None,
+                        username_override: None,
+                    })
+                } else {
+                    None
+                }
+            }
+        }
+    }
+}
+
+async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
+    let auth_header = parts
+        .headers
+        .get(http::header::AUTHORIZATION)
+        .and_then(|value| value.to_str().ok())
+        .and_then(|s| s.strip_prefix("Bearer "));
+
+    let from_cookie = match auth_header {
+        Some(x) => Some(x.to_owned()),
+        None => Extension::<Cookies>::from_request_parts(parts, state)
+            .await
+            .ok()
+            .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
+    };
+
+    #[derive(Deserialize)]
+    struct Token {
+        token: Option<String>,
+    }
+    match from_cookie {
+        Some(token) => Some(token),
+        None => Query::<Token>::from_request_parts(parts, state)
+            .await
+            .ok()
+            .and_then(|token| token.token.clone()),
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct Tokened {
+    pub token: String,
+}
+
+pub struct OptTokened {
+    #[allow(dead_code)]
+    pub token: Option<String>,
+}
+
+struct BruteForceCounter {
+    counter: AtomicU64,
+    last_reset: AtomicI64,
+}
+
+lazy_static::lazy_static! {
+    static ref BRUTE_FORCE_COUNTER: BruteForceCounter =
+        BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) };
+}
+
+impl BruteForceCounter {
+    async fn increment(&self) {
+        let now = time::OffsetDateTime::now_utc().unix_timestamp();
+        if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 {
+            tracing::error!(
+                "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds"
+            );
+            tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+        }
+        if now - self.last_reset.load(Ordering::Relaxed) > 60 {
+            self.counter.store(0, Ordering::Relaxed);
+            self.last_reset.store(now, Ordering::Relaxed);
+        }
+    }
+}
+
+#[async_trait]
+impl<S> FromRequestParts<S> for Tokened
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &S,
+    ) -> std::result::Result<Self, Self::Rejection> {
+        if parts.method == http::Method::OPTIONS {
+            return Ok(Tokened { token: "".to_string() });
+        };
+        let already_tokened = parts.extensions.get::<Tokened>();
+        if let Some(tokened) = already_tokened {
+            Ok(tokened.clone())
+        } else {
+            let token_o = extract_token(parts, state).await;
+            if let Some(token) = token_o {
+                let tokened = Self { token };
+                parts.extensions.insert(tokened.clone());
+                Ok(tokened)
+            } else {
+                BRUTE_FORCE_COUNTER.increment().await;
+                Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl<S> FromRequestParts<S> for OptTokened
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &S,
+    ) -> std::result::Result<Self, Self::Rejection> {
+        if parts.method == http::Method::OPTIONS {
+            return Ok(OptTokened { token: None });
+        };
+        let already_tokened = parts.extensions.get::<Tokened>();
+        if let Some(tokened) = already_tokened {
+            Ok(OptTokened { token: Some(tokened.token.clone()) })
+        } else {
+            let token_o = extract_token(parts, state).await;
+            Ok(OptTokened { token: token_o })
+        }
+    }
+}
+
+#[async_trait]
+impl<S> FromRequestParts<S> for ApiAuthed
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(
+        parts: &mut Parts,
+        state: &S,
+    ) -> std::result::Result<Self, Self::Rejection> {
+        if parts.method == http::Method::OPTIONS {
+            return Ok(ApiAuthed {
+                email: "".to_owned(),
+                username: "".to_owned(),
+                is_admin: false,
+                is_operator: false,
+                groups: Vec::new(),
+                folders: Vec::new(),
+                scopes: None,
+                username_override: None,
+            });
+        };
+        let already_authed = parts.extensions.get::<ApiAuthed>();
+        if let Some(authed) = already_authed {
+            Ok(authed.clone())
+        } else {
+            let already_tokened = parts.extensions.get::<Tokened>();
+            let token_o = if let Some(token) = already_tokened {
+                Some(token.token.clone())
+            } else {
+                extract_token(parts, state).await
+            };
+            let original_uri = OriginalUri::from_request_parts(parts, state)
+                .await
+                .ok()
+                .map(|x| x.0)
+                .unwrap_or_default();
+            let path_vec: Vec<&str> = original_uri.path().split("/").collect();
+
+            let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
+                Some(path_vec[3].to_owned())
+            } else {
+                if path_vec.len() >= 5
+                    && path_vec[0] == ""
+                    && path_vec[2] == "srch"
+                    && path_vec[3] == "w"
+                {
+                    Some(path_vec[4].to_string())
+                } else {
+                    None
+                }
+            };
+            if let Some(token) = token_o {
+                if let Ok(Extension(cache)) =
+                    Extension::<Arc<AuthCache>>::from_request_parts(parts, state).await
+                {
+                    if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
+                        parts.extensions.insert(authed.clone());
+                        if authed.scopes.as_ref().is_some_and(|scopes| {
+                            scopes
+                                .iter()
+                                .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
+                        }) && (path_vec.len() < 3
+                            || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
+                        {
+                            BRUTE_FORCE_COUNTER.increment().await;
+                            return Err((
+                                StatusCode::UNAUTHORIZED,
+                                format!("Unauthorized scoped token: {:?}", authed.scopes),
+                            ));
+                        }
+                        Span::current().record("username", &authed.username.as_str());
+                        Span::current().record("email", &authed.email);
+
+                        if let Some(workspace_id) = workspace_id {
+                            Span::current().record("workspace_id", &workspace_id);
+                        }
+                        return Ok(authed);
+                    }
+                }
+            }
+            BRUTE_FORCE_COUNTER.increment().await;
+            Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
+        }
+    }
+}
+
+fn username_override_from_label(label: Option<String>) -> Option<String> {
+    match label {
+        Some(label)
+            if label.starts_with("webhook-")
+                || label.starts_with("http-")
+                || label.starts_with("email-")
+                || label.starts_with("ws-") =>
+        {
+            Some(label)
+        }
+        Some(label) if label.starts_with("ephemeral-script-end-user-") => Some(
+            label
+                .trim_start_matches("ephemeral-script-end-user-")
+                .to_string(),
+        ),
+        Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()),
+        Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => {
+            Some(format!("label-{label}"))
+        }
+        _ => None,
+    }
+}

backend/windmill-api/src/folders.rs

@@ -11,8 +11,9 @@ use std::sync::Arc;
 use crate::db::ApiAuthed;
 
 use crate::{
+    auth::AuthCache,
     db::DB,
-    users::{AuthCache, Tokened},
+    users::Tokened,
     webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{

backend/windmill-api/src/http_triggers.rs

@@ -2,12 +2,13 @@
 use crate::job_helpers_ee::get_workspace_s3_resource;
 use crate::{
     args::WebhookArgs,
+    auth::{AuthCache, OptTokened},
     db::{ApiAuthed, DB},
     jobs::{
         run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal,
         run_wait_result_script_by_path_internal, RunJobQuery,
     },
-    users::{fetch_api_authed, AuthCache, OptTokened},
+    users::fetch_api_authed,
 };
 use axum::{
     extract::{Path, Query},

backend/windmill-api/src/job_helpers_ee.rs

@@ -12,6 +12,12 @@ use std::sync::Arc;
 use windmill_common::error;
 #[cfg(feature = "parquet")]
 use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource};
+
+#[cfg(feature = "parquet")]
+use bytes::Bytes;
+#[cfg(feature = "parquet")]
+use futures::Stream;
+
 #[derive(Serialize)]
 pub struct UploadFileResponse {
     pub file_key: String,
@@ -35,7 +41,7 @@ pub async fn get_workspace_s3_resource<'c>(
 }
 
 pub fn get_random_file_name(_file_extension: Option<String>) -> String {
-    todo!()
+    unimplemented!("Not implemented in Windmill's Open Source repository")
 }
 
 pub async fn get_s3_resource<'c>(
@@ -48,14 +54,31 @@ pub async fn get_s3_resource<'c>(
     _resource_type: Option<StorageResourceType>,
     _job_id: Option<Uuid>,
 ) -> error::Result<ObjectStoreResource> {
-    todo!()
+    Err(error::Error::InternalErr(
+        "Not implemented in Windmill's Open Source repository".to_string(),
+    ))
 }
 
+#[cfg(feature = "parquet")]
+pub async fn upload_file_from_req(
+    _s3_client: Arc<dyn ObjectStore>,
+    _file_key: &str,
+    _req: axum::extract::Request,
+    _options: PutMultipartOpts,
+) -> error::Result<()> {
+    Err(error::Error::InternalErr(
+        "Not implemented in Windmill's Open Source repository".to_string(),
+    ))
+}
+
+#[cfg(feature = "parquet")]
 pub async fn upload_file_internal(
     _s3_client: Arc<dyn ObjectStore>,
     _file_key: &str,
-    _request: axum::extract::Request,
+    _stream: impl Stream<Item = Result<Bytes, std::io::Error>> + Unpin,
     _options: PutMultipartOpts,
 ) -> error::Result<()> {
-    todo!()
+    Err(error::Error::InternalErr(
+        "Not implemented in Windmill's Open Source repository".to_string(),
+    ))
 }

backend/windmill-api/src/jobs.rs

@@ -8,6 +8,7 @@
 
 use axum::body::Body;
 use axum::http::HeaderValue;
+use futures::TryFutureExt;
 use itertools::Itertools;
 use quick_cache::sync::Cache;
 use serde_json::value::RawValue;
@@ -62,6 +63,7 @@ use windmill_audit::audit_ee::{audit_log, AuditAuthor};
 use windmill_audit::ActionKind;
 use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE};
 use windmill_common::{
+    cache,
     db::UserDB,
     error::{self, to_anyhow, Error},
     flow_status::{Approval, FlowStatus, FlowStatusModule},
@@ -776,7 +778,7 @@ impl<'a> GetQuery<'a> {
         Self { with_in_tags: in_tags, ..self }
     }
 
-    fn check_auth(self, email: Option<&str>) -> error::Result<()> {
+    fn check_auth(&self, email: Option<&str>) -> error::Result<()> {
         if let Some(email) = email {
             if self.with_auth.is_some_and(|x| x.is_none()) && email != "anonymous" {
                 return Err(Error::BadRequest(
@@ -788,6 +790,53 @@ impl<'a> GetQuery<'a> {
         Ok(())
     }
 
+    /// Resolve job raw values.
+    /// This fetch the raw values from the cache and update the job accordingly.
+    ///
+    /// # Details
+    /// Most of the raw values (code, lock and flow) had been removed from the `job`, `queue` and
+    /// `completed_job` tables. Only remains ones for "preview" jobs (i.e. [`JobKind::Preview`],
+    /// [`JobKind::FlowPreview`] and [`JobKind::Dependencies`]). [`JobKind::Flow`] as well but only
+    /// when pushed from an un-updated workers.
+    /// This function is used to make the above change transparent for the API, as the returned jobs
+    /// will have the raw values as if they were still in the tables.
+    async fn resolve_raw_values<T>(
+        &self,
+        db: &DB,
+        id: Uuid,
+        kind: JobKind,
+        hash: Option<ScriptHash>,
+        job: &mut JobExtended<T>,
+    ) {
+        let (raw_code, raw_lock, raw_flow) = (
+            job.raw_code.take(),
+            job.raw_lock.take(),
+            job.raw_flow.take(),
+        );
+        if self.with_flow {
+            // Try to fetch the flow from the cache, fallback to the preview flow.
+            // NOTE: This could check for the job kinds instead of the `or_else` but it's not
+            // necessary as `fetch_flow` return early if the job kind is not a preview one.
+            cache::job::fetch_flow(db, kind, hash)
+                .or_else(|_| cache::job::fetch_preview_flow(db, &id, raw_flow))
+                .await
+                .ok()
+                .inspect(|data| job.raw_flow = Some(sqlx::types::Json(data.raw_flow.clone())));
+        }
+        if self.with_code {
+            // Try to fetch the code from the cache, fallback to the preview code.
+            // NOTE: This could check for the job kinds instead of the `or_else` but it's not
+            // necessary as `fetch_script` return early if the job kind is not a preview one.
+            cache::job::fetch_script(db, kind, hash)
+                .or_else(|_| cache::job::fetch_preview_script(db, &id, raw_lock, raw_code))
+                .await
+                .ok()
+                .inspect(|data| {
+                    (job.raw_lock, job.raw_code) = (data.lock.clone(), Some(data.code.clone()))
+                });
+        }
+    }
+
     async fn fetch_queued(
         self,
         db: &DB,
@@ -807,6 +856,10 @@ impl<'a> GetQuery<'a> {
         let mut job = query.fetch_optional(db).await?;
 
         self.check_auth(job.as_ref().map(|job| job.created_by.as_str()))?;
+        if let Some(job) = job.as_mut() {
+            self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
+                .await;
+        }
         if self.with_flow {
             job = resolve_maybe_value(db, workspace_id, self.with_code, job, |job| {
                 job.raw_flow.as_mut()
@@ -835,6 +888,10 @@ impl<'a> GetQuery<'a> {
         let mut cjob = query.fetch_optional(db).await?;
 
         self.check_auth(cjob.as_ref().map(|job| job.created_by.as_str()))?;
+        if let Some(job) = cjob.as_mut() {
+            self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
+                .await;
+        }
         if self.with_flow {
             cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| {
                 job.raw_flow.as_mut()
@@ -2103,7 +2160,7 @@ pub struct SuspendedJobFlow {
     pub approvers: Vec<Approval>,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Debug)]
 pub struct QueryApprover {
     pub approver: Option<String>,
 }
@@ -2316,11 +2373,11 @@ fn create_signature(
 }
 
 #[allow(non_snake_case)]
-#[derive(Serialize)]
+#[derive(Serialize, Debug)]
 pub struct ResumeUrls {
-    approvalPage: String,
-    cancel: String,
-    resume: String,
+    pub approvalPage: String,
+    pub cancel: String,
+    pub resume: String,
 }
 
 fn build_resume_url(
@@ -2340,6 +2397,14 @@ pub async fn get_resume_urls(
     Extension(db): Extension<DB>,
     Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
     Query(approver): Query<QueryApprover>,
+) -> error::JsonResult<ResumeUrls> {
+    get_resume_urls_internal(Extension(db), Path((w_id, job_id, resume_id)), Query(approver)).await
+}
+
+pub async fn get_resume_urls_internal(
+    Extension(db): Extension<DB>,
+    Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
+    Query(approver): Query<QueryApprover>,
 ) -> error::JsonResult<ResumeUrls> {
     let key = get_workspace_key(&w_id, &db).await?;
     let signature = create_signature(key, job_id, resume_id, approver.approver.clone())?;
@@ -2487,10 +2552,7 @@ impl Job {
     }
 
     pub fn is_flow(&self) -> bool {
-        matches!(
-            self.job_kind(),
-            JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode
-        )
+        self.job_kind().is_flow()
     }
 
     pub fn job_kind(&self) -> &JobKind {
@@ -5507,3 +5569,4 @@ async fn delete_completed_job<'a>(
     let response = Json(cj).into_response();
     Ok(response)
 }
+

backend/windmill-api/src/lib.rs

@@ -11,23 +11,30 @@ use crate::db::ApiAuthed;
 use crate::ee::ExternalJwks;
 #[cfg(feature = "embedding")]
 use crate::embeddings::load_embeddings_db;
+#[cfg(feature = "oauth2")]
 use crate::oauth2_ee::AllClients;
+#[cfg(feature = "oauth2")]
+use crate::oauth2_ee::SlackVerifier;
+#[cfg(feature = "smtp")]
 use crate::smtp_server_ee::SmtpServer;
+
 use crate::tracing_init::MyOnFailure;
 use crate::{
-    oauth2_ee::SlackVerifier,
     tracing_init::{MyMakeSpan, MyOnResponse},
     users::OptAuthed,
     webhook_util::WebhookShared,
 };
+
 use anyhow::Context;
 use argon2::Argon2;
 use axum::extract::DefaultBodyLimit;
-use axum::{middleware::from_extractor, routing::get, Extension, Router};
+use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
 use db::DB;
 use http::HeaderValue;
 use reqwest::Client;
+#[cfg(feature = "oauth2")]
 use std::collections::HashMap;
+
 use std::time::Duration;
 use std::{net::SocketAddr, sync::Arc};
 use tokio::sync::RwLock;
@@ -48,6 +55,7 @@ mod ai;
 mod apps;
 mod args;
 mod audit;
+mod auth;
 mod capture;
 mod concurrency_groups;
 mod configs;
@@ -60,6 +68,7 @@ mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
+#[cfg(feature = "http_trigger")]
 mod http_triggers;
 mod indexer_ee;
 mod inputs;
@@ -73,6 +82,7 @@ pub mod job_metrics;
 pub mod jobs;
 #[cfg(all(feature = "enterprise", feature = "kafka"))]
 mod kafka_triggers_ee;
+#[cfg(feature = "oauth2")]
 pub mod oauth2_ee;
 mod oidc_ee;
 mod raw_apps;
@@ -83,7 +93,8 @@ mod scim_ee;
 mod scripts;
 mod service_logs;
 mod settings;
-pub mod smtp_server_ee;
+#[cfg(feature = "smtp")]
+mod smtp_server_ee;
 mod static_assets;
 mod stripe_ee;
 mod tracing_init;
@@ -93,10 +104,14 @@ mod users_ee;
 mod utils;
 mod variables;
 mod webhook_util;
+#[cfg(feature = "websocket")]
 mod websocket_triggers;
 mod workers;
 mod workspaces;
 mod workspaces_ee;
+mod slack_approvals;
+mod workspaces_export;
+mod workspaces_extra;
 
 pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
 
@@ -110,10 +125,6 @@ lazy_static::lazy_static! {
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
 
-    pub static ref SLACK_SIGNING_SECRET: Option<SlackVerifier> = std::env::var("SLACK_SIGNING_SECRET")
-        .ok()
-        .map(|x| SlackVerifier::new(x).unwrap());
-
     pub static ref IS_SECURE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
 
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
@@ -123,11 +134,22 @@ lazy_static::lazy_static! {
         .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
         .build().unwrap();
 
+
+}
+
+#[cfg(feature = "oauth2")]
+lazy_static::lazy_static! {
     pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
         logins: HashMap::new(),
         connects: HashMap::new(),
         slack: None
     }));
+
+
+    pub static ref SLACK_SIGNING_SECRET: Option<SlackVerifier> = std::env::var("SLACK_SIGNING_SECRET")
+        .ok()
+        .map(|x| SlackVerifier::new(x).unwrap());
+
 }
 
 // Compliance with cloud events spec.
@@ -171,13 +193,13 @@ pub async fn run_server(
     mut rx: tokio::sync::broadcast::Receiver<()>,
     port_tx: tokio::sync::oneshot::Sender<String>,
     server_mode: bool,
-    base_internal_url: String,
+    #[cfg(feature = "smtp")] base_internal_url: String,
 ) -> anyhow::Result<()> {
     let user_db = UserDB::new(db.clone());
 
     #[cfg(feature = "enterprise")]
     let ext_jwks = ExternalJwks::load().await;
-    let auth_cache = Arc::new(users::AuthCache::new(
+    let auth_cache = Arc::new(crate::auth::AuthCache::new(
         db.clone(),
         std::env::var("SUPERADMIN_SECRET").ok(),
         #[cfg(feature = "enterprise")]
@@ -214,14 +236,17 @@ pub async fn run_server(
         #[cfg(feature = "embedding")]
         load_embeddings_db(&db);
 
-        let smtp_server = Arc::new(SmtpServer {
-            db: db.clone(),
-            user_db: user_db,
-            auth_cache: auth_cache.clone(),
-            base_internal_url: base_internal_url.clone(),
-        });
-        if let Err(err) = smtp_server.start_listener_thread(addr).await {
-            tracing::error!("Error starting SMTP server: {err:#}");
+        #[cfg(feature = "smtp")]
+        {
+            let smtp_server = Arc::new(SmtpServer {
+                db: db.clone(),
+                user_db: user_db,
+                auth_cache: auth_cache.clone(),
+                base_internal_url: base_internal_url.clone(),
+            });
+            if let Err(err) = smtp_server.start_listener_thread(addr).await {
+                tracing::error!("Error starting SMTP server: {err:#}");
+            }
         }
     }
 
@@ -253,8 +278,11 @@ pub async fn run_server(
     };
 
     if !*CLOUD_HOSTED {
-        let ws_killpill_rx = rx.resubscribe();
-        websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await;
+        #[cfg(feature = "websocket")]
+        {
+            let ws_killpill_rx = rx.resubscribe();
+            websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await;
+        }
 
         #[cfg(all(feature = "enterprise", feature = "kafka"))]
         {
@@ -290,7 +318,15 @@ pub async fn run_server(
                         .nest("/job_metrics", job_metrics::workspaced_service())
                         .nest("/job_helpers", job_helpers_service)
                         .nest("/jobs", jobs::workspaced_service())
-                        .nest("/oauth", oauth2_ee::workspaced_service())
+                        .nest("/oauth", {
+                            #[cfg(feature = "oauth2")]
+                            {
+                                oauth2_ee::workspaced_service()
+                            }
+
+                            #[cfg(not(feature = "oauth2"))]
+                            Router::new()
+                        })
                         .nest("/ai", ai::workspaced_service())
                         .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/resources", resources::workspaced_service())
@@ -303,11 +339,24 @@ pub async fn run_server(
                         .nest("/variables", variables::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
                         .nest("/oidc", oidc_ee::workspaced_service())
-                        .nest("/http_triggers", http_triggers::workspaced_service())
-                        .nest(
-                            "/websocket_triggers",
-                            websocket_triggers::workspaced_service(),
-                        )
+                        .nest("/http_triggers", {
+                            #[cfg(feature = "http_trigger")]
+                            {
+                                http_triggers::workspaced_service()
+                            }
+
+                            #[cfg(not(feature = "http_trigger"))]
+                            Router::new()
+                        })
+                        .nest("/websocket_triggers", {
+                            #[cfg(feature = "websocket")]
+                            {
+                                websocket_triggers::workspaced_service()
+                            }
+
+                            #[cfg(not(feature = "websocket"))]
+                            Router::new()
+                        })
                         .nest("/kafka_triggers", kafka_triggers_service),
                 )
                 .nest("/workspaces", workspaces::global_service())
@@ -367,6 +416,8 @@ pub async fn run_server(
                     "/w/:workspace_id/jobs_u",
                     jobs::workspace_unauthed_service().layer(cors.clone()),
                 )
+                .route("/slack", post(slack_approvals::slack_app_callback_handler))
+                .route("/w/:workspace_id/jobs/slack_approval/:job_id", get(slack_approvals::request_slack_approval))
                 .nest(
                     "/w/:workspace_id/resources_u",
                     resources::public_service().layer(cors.clone()),
@@ -379,13 +430,29 @@ pub async fn run_server(
                     "/auth",
                     users::make_unauthed_service().layer(Extension(argon2)),
                 )
-                .nest(
-                    "/oauth",
-                    oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))),
-                )
+                .nest("/oauth", {
+                    #[cfg(feature = "oauth2")]
+                    {
+                        oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension)))
+                    }
+
+                    #[cfg(not(feature = "oauth2"))]
+                    Router::new()
+                })
                 .nest(
                     "/r",
-                    http_triggers::routes_global_service().layer(from_extractor::<OptAuthed>()),
+                    {
+                        #[cfg(feature = "http_trigger")]
+                        {
+                            http_triggers::routes_global_service()
+                        }
+
+                        #[cfg(not(feature = "http_trigger"))]
+                        {
+                            Router::new()
+                        }
+                    }
+                    .layer(from_extractor::<OptAuthed>()),
                 )
                 .route("/version", get(git_v))
                 .route("/uptodate", get(is_up_to_date))

backend/windmill-api/src/oauth2_ee.rs

@@ -10,14 +10,17 @@ use std::{collections::HashMap, fmt::Debug};
 
 use axum::{routing::get, Json, Router};
 use hmac::Mac;
-use hyper::HeaderMap;
 
+#[cfg(feature = "oauth2")]
 use itertools::Itertools;
+#[cfg(feature = "oauth2")]
 use oauth2::{Client as OClient, *};
 use serde::{Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
+#[cfg(feature = "oauth2")]
 use windmill_common::more_serde::maybe_number_opt;
 
+#[cfg(feature = "oauth2")]
 use crate::OAUTH_CLIENTS;
 use windmill_common::error;
 use windmill_common::oauth2::*;
@@ -27,7 +30,6 @@ use std::str;
 
 pub fn global_service() -> Router {
     Router::new()
-        .route("/list_supabase", get(list_supabase))
         .route("/list_logins", get(list_logins))
         .route("/list_connects", get(list_connects))
 }
@@ -36,17 +38,7 @@ pub fn workspaced_service() -> Router {
     Router::new()
 }
 
-#[derive(Serialize)]
-#[serde(tag = "type")]
-pub enum InstanceEvent {
-    UserAdded { email: String },
-    // UserDeleted { email: String },
-    // UserDeletedWorkspace { workspace: String, email: String },
-    UserAddedWorkspace { workspace: String, email: String },
-    UserInvitedWorkspace { workspace: String, email: String },
-    UserJoinedWorkspace { workspace: String, email: String, username: String },
-}
-
+#[cfg(feature = "oauth2")]
 #[derive(Debug, Clone)]
 pub struct ClientWithScopes {
     _client: OClient,
@@ -56,7 +48,7 @@ pub struct ClientWithScopes {
     _allowed_domains: Option<Vec<String>>,
     _userinfo_url: Option<String>,
 }
-
+#[cfg(feature = "oauth2")]
 pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -79,6 +71,7 @@ pub struct OAuthClient {
     login_config: Option<OAuthConfig>,
 }
 
+#[cfg(feature = "oauth2")]
 #[derive(Debug)]
 pub struct AllClients {
     pub logins: BasicClientsMap,
@@ -86,6 +79,7 @@ pub struct AllClients {
     pub slack: Option<OClient>,
 }
 
+#[cfg(feature = "oauth2")]
 pub fn build_oauth_clients(
     _base_url: &str,
     _oauths_from_config: Option<HashMap<String, OAuthClient>>,
@@ -98,6 +92,7 @@ pub fn build_oauth_clients(
     });
 }
 
+#[cfg(feature = "oauth2")]
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct TokenResponse {
     access_token: AccessToken,
@@ -121,6 +116,7 @@ async fn list_logins() -> error::JsonResult<Logins> {
     return Ok(Json(Logins { oauth: vec![], saml: None }));
 }
 
+#[cfg(feature = "oauth2")]
 async fn list_connects() -> error::JsonResult<Vec<String>> {
     Ok(Json(
         (&OAUTH_CLIENTS.read().await.connects)
@@ -130,6 +126,12 @@ async fn list_connects() -> error::JsonResult<Vec<String>> {
     ))
 }
 
+#[cfg(not(feature = "oauth2"))]
+async fn list_connects() -> error::JsonResult<Vec<String>> {
+    // Implementation is not open source
+    return Ok(Json(vec![]));
+}
+
 pub async fn _refresh_token<'c>(
     _tx: Transaction<'c, Postgres>,
     _path: &str,
@@ -143,13 +145,6 @@ pub async fn _refresh_token<'c>(
     ))
 }
 
-async fn list_supabase(_headers: HeaderMap) -> error::Result<String> {
-    // Implementation is not open source
-    Err(error::Error::BadRequest(
-        "Not implemented in Windmill's Open Source repository".to_string(),
-    ))
-}
-
 pub async fn check_nb_of_user(db: &DB) -> error::Result<()> {
     let nb_users_sso =
         sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)

backend/windmill-api/src/scripts.rs

@@ -12,7 +12,8 @@ use crate::{
     triggers::{
         get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail,
     },
-    users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
+    users::{maybe_refresh_folders, require_owner_of_path},
+    auth::AuthCache,
     utils::WithStarredInfoQuery,
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,

backend/windmill-api/src/smtp_server_ee.rs

@@ -1,4 +1,4 @@
-use crate::{db::DB, users::AuthCache};
+use crate::{auth::AuthCache, db::DB};
 use std::{net::SocketAddr, sync::Arc};
 use windmill_common::db::UserDB;
 
@@ -11,6 +11,10 @@ pub struct SmtpServer {
 
 impl SmtpServer {
     pub async fn start_listener_thread(self: Arc<Self>, _addr: SocketAddr) -> anyhow::Result<()> {
+        let _ = self.auth_cache;
+        let _ = self.db;
+        let _ = self.user_db;
+        let _ = self.base_internal_url;
         Err(anyhow::anyhow!("Implementation not open source"))
     }
 }

backend/windmill-api/src/static_assets.rs

@@ -6,15 +6,15 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use axum::{
-    body::Body,
-    extract::OriginalUri,
-    http::{header, Response},
-    response::IntoResponse,
-};
+use axum::{body::Body, extract::OriginalUri, http::Response, response::IntoResponse};
+
+#[cfg(feature = "static_frontend")]
+use axum::http::header;
 
 use hyper::Uri;
+#[cfg(feature = "static_frontend")]
 use mime_guess::mime;
+#[cfg(feature = "static_frontend")]
 use rust_embed::RustEmbed;
 
 // static_handler is a handler that serves static files from the
@@ -22,6 +22,7 @@ pub async fn static_handler(OriginalUri(original_uri): OriginalUri) -> StaticFil
     StaticFile(original_uri)
 }
 
+#[cfg(feature = "static_frontend")]
 #[derive(RustEmbed)]
 #[folder = "${FRONTEND_BUILD_DIR:-../../frontend/build/}"]
 struct Asset;
@@ -34,6 +35,7 @@ impl IntoResponse for StaticFile {
     }
 }
 
+#[cfg(feature = "static_frontend")]
 const TWO_HUNDRED: &str = "200.html";
 
 fn serve_path(path: &str) -> Response<Body> {
@@ -41,6 +43,7 @@ fn serve_path(path: &str) -> Response<Body> {
         return Response::builder().status(404).body(Body::empty()).unwrap();
     }
 
+    #[cfg(feature = "static_frontend")]
     match Asset::get(path) {
         Some(content) => {
             let body = Body::from(content.data);
@@ -68,4 +71,9 @@ fn serve_path(path: &str) -> Response<Body> {
         }
         None => serve_path(TWO_HUNDRED),
     }
+
+    #[cfg(not(feature = "static_frontend"))]
+    {
+        Response::builder().status(404).body(Body::empty()).unwrap()
+    }
 }

backend/windmill-api/src/users.rs

@@ -8,14 +8,13 @@
 
 #![allow(non_snake_case)]
 
-use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
+use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
 use crate::db::ApiAuthed;
 
-#[cfg(feature = "enterprise")]
-use crate::ee::ExternalJwks;
-use crate::oauth2_ee::InstanceEvent;
+pub use crate::auth::Tokened;
+
 use crate::utils::{
     generate_instance_wide_unique_username, get_instance_username_or_create_pending,
 };
@@ -25,43 +24,38 @@ use crate::{
 use argon2::{Argon2, PasswordHash, PasswordVerifier};
 use axum::{
     async_trait,
-    extract::{Extension, FromRequestParts, OriginalUri, Path, Query},
+    extract::{Extension, FromRequestParts, Path, Query},
     http::request::Parts,
     response::{IntoResponse, Response},
     routing::{delete, get, post},
     Json, Router,
 };
-use chrono::TimeZone;
 use hyper::{header::LOCATION, StatusCode};
 use lazy_static::lazy_static;
-use quick_cache::sync::Cache;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use time::OffsetDateTime;
-#[cfg(feature = "enterprise")]
-use tokio::sync::RwLock;
 use tower_cookies::{Cookie, Cookies};
-use tracing::{Instrument, Span};
+use tracing::Instrument;
 use windmill_audit::audit_ee::{audit_log, AuditAuthor};
 use windmill_audit::ActionKind;
 use windmill_common::auth::fetch_authed_from_permissioned_as;
 use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING;
+use windmill_common::oauth2::InstanceEvent;
+use windmill_common::users::COOKIE_NAME;
 use windmill_common::users::{truncate_token, username_to_permissioned_as};
 use windmill_common::utils::paginate;
 use windmill_common::worker::CLOUD_HOSTED;
 use windmill_common::{
-    auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET},
+    auth::{get_folders_for_user, get_groups_for_user},
     db::UserDB,
     error::{self, Error, JsonResult, Result},
-    users::SUPERADMIN_SECRET_EMAIL,
     utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
 };
 use windmill_git_sync::handle_deployment_metadata;
-
 pub const TTL_TOKEN_DB_H: u32 = 72;
 
-const COOKIE_NAME: &str = "token";
 const COOKIE_PATH: &str = "/";
 
 pub fn workspaced_service() -> Router {
@@ -126,478 +120,6 @@ pub fn make_unauthed_service() -> Router {
         .route("/is_first_time_setup", get(is_first_time_setup))
 }
 
-fn username_override_from_label(label: Option<String>) -> Option<String> {
-    match label {
-        Some(label)
-            if label.starts_with("webhook-")
-                || label.starts_with("http-")
-                || label.starts_with("email-")
-                || label.starts_with("ws-") =>
-        {
-            Some(label)
-        }
-        Some(label) if label.starts_with("ephemeral-script-end-user-") => Some(
-            label
-                .trim_start_matches("ephemeral-script-end-user-")
-                .to_string(),
-        ),
-        Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()),
-        Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => {
-            Some(format!("label-{label}"))
-        }
-        _ => None,
-    }
-}
-
-#[derive(Clone)]
-pub struct ExpiringAuthCache {
-    pub authed: ApiAuthed,
-    pub expiry: chrono::DateTime<chrono::Utc>,
-}
-
-pub struct AuthCache {
-    cache: Cache<(String, String), ExpiringAuthCache>,
-    db: DB,
-    superadmin_secret: Option<String>,
-    #[cfg(feature = "enterprise")]
-    ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
-}
-
-impl AuthCache {
-    pub fn new(
-        db: DB,
-        superadmin_secret: Option<String>,
-        #[cfg(feature = "enterprise")] ext_jwks: Option<Arc<RwLock<ExternalJwks>>>,
-    ) -> Self {
-        AuthCache {
-            cache: Cache::new(300),
-            db,
-            superadmin_secret,
-            #[cfg(feature = "enterprise")]
-            ext_jwks,
-        }
-    }
-
-    pub async fn invalidate(&self, w_id: &str, token: String) {
-        self.cache.remove(&(w_id.to_string(), token));
-    }
-
-    pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
-        let key = (
-            w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
-            token.to_string(),
-        );
-        let s = self.cache.get(&key).map(|c| c.to_owned());
-        match s {
-            Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => {
-                Some(authed)
-            }
-            #[cfg(feature = "enterprise")]
-            _ if token.starts_with("jwt_ext_") => {
-                let authed_and_exp = match crate::ee::jwt_ext_auth(
-                    w_id.as_ref(),
-                    token.trim_start_matches("jwt_ext_"),
-                    self.ext_jwks.clone(),
-                )
-                .await
-                {
-                    Ok(r) => Some(r),
-                    Err(e) => {
-                        tracing::error!("JWT_EXT auth error: {:?}", e);
-                        None
-                    }
-                };
-
-                if let Some((authed, exp)) = authed_and_exp.clone() {
-                    self.cache.insert(
-                        key,
-                        ExpiringAuthCache {
-                            authed: authed.clone(),
-                            expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000),
-                        },
-                    );
-
-                    Some(authed)
-                } else {
-                    None
-                }
-            }
-            _ if token.starts_with("jwt_") => {
-                let jwt_secret = JWT_SECRET.read().await;
-                if !jwt_secret.is_empty() {
-                    let jwt_token = token.trim_start_matches("jwt_");
-
-                    let jwt_result = jsonwebtoken::decode::<JWTAuthClaims>(
-                        jwt_token,
-                        &jsonwebtoken::DecodingKey::from_secret(jwt_secret.as_bytes()),
-                        &jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256),
-                    );
-
-                    match jwt_result {
-                        Ok(payload) => {
-                            if w_id.is_some_and(|w_id| w_id != payload.claims.workspace_id) {
-                                tracing::error!("JWT auth error: workspace_id mismatch");
-                                return None;
-                            }
-
-                            let username_override =
-                                username_override_from_label(payload.claims.label);
-                            let authed = crate::db::ApiAuthed {
-                                email: payload.claims.email,
-                                username: payload.claims.username,
-                                is_admin: payload.claims.is_admin,
-                                is_operator: payload.claims.is_operator,
-                                groups: payload.claims.groups,
-                                folders: payload.claims.folders,
-                                scopes: None,
-                                username_override,
-                            };
-
-                            self.cache.insert(
-                                key,
-                                ExpiringAuthCache {
-                                    authed: authed.clone(),
-                                    expiry: chrono::Utc
-                                        .timestamp_nanos(payload.claims.exp as i64 * 1_000_000_000),
-                                },
-                            );
-
-                            Some(authed)
-                        }
-                        Err(err) => {
-                            tracing::error!("JWT auth error: {:?}", err);
-                            None
-                        }
-                    }
-                } else {
-                    tracing::error!("JWT auth error: no jwt secret set");
-                    None
-                }
-            }
-            _ => {
-                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>, Option<String>)>(
-                    "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
-                     OR expiration IS NULL) AND (workspace_id IS NULL OR workspace_id = $2) RETURNING owner, email, super_admin, scopes, label",
-                )
-                .bind(token)
-                .bind(w_id.as_ref())
-                .fetch_optional(&self.db)
-                .await
-                .ok()
-                .flatten();
-
-                if let Some(user) = user_o {
-                    let authed_o = {
-                        match user {
-                            (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
-                                let username_override = username_override_from_label(label);
-                                if let Some((prefix, name)) = owner.split_once('/') {
-                                    if prefix == "u" {
-                                        let (is_admin, is_operator) = if super_admin {
-                                            (true, false)
-                                        } else {
-                                            let r = sqlx::query!(
-                                                "SELECT is_admin, operator FROM usr where username = $1 AND \
-                                                 workspace_id = $2 AND disabled = false",
-                                                name,
-                                                &w_id.as_ref().unwrap()
-                                            )
-                                            .fetch_one(&self.db)
-                                            .await
-                                            .ok();
-                                            if let Some(r) = r {
-                                                (r.is_admin, r.operator)
-                                            } else {
-                                                (false, true)
-                                            }
-                                        };
-
-                                        let w_id = &w_id.unwrap();
-                                        let groups =
-                                            get_groups_for_user(w_id, &name, &email, &self.db)
-                                                .await
-                                                .ok()
-                                                .unwrap_or_default();
-
-                                        let folders =
-                                            get_folders_for_user(w_id, &name, &groups, &self.db)
-                                                .await
-                                                .ok()
-                                                .unwrap_or_default();
-
-                                        Some(ApiAuthed {
-                                            email: email,
-                                            username: name.to_string(),
-                                            is_admin,
-                                            is_operator,
-                                            groups,
-                                            folders,
-                                            scopes: None,
-                                            username_override,
-                                        })
-                                    } else {
-                                        let groups = vec![name.to_string()];
-                                        let folders = get_folders_for_user(
-                                            &w_id.unwrap(),
-                                            "",
-                                            &groups,
-                                            &self.db,
-                                        )
-                                        .await
-                                        .ok()
-                                        .unwrap_or_default();
-                                        Some(ApiAuthed {
-                                            email: email,
-                                            username: format!("group-{name}"),
-                                            is_admin: false,
-                                            groups,
-                                            is_operator: false,
-                                            folders,
-                                            scopes: None,
-                                            username_override,
-                                        })
-                                    }
-                                } else {
-                                    let groups = vec![];
-                                    let folders = vec![];
-                                    Some(ApiAuthed {
-                                        email: email,
-                                        username: owner,
-                                        is_admin: super_admin,
-                                        is_operator: true,
-                                        groups,
-                                        folders,
-                                        scopes: None,
-                                        username_override,
-                                    })
-                                }
-                            }
-                            (_, Some(email), super_admin, scopes, label) => {
-                                let username_override = username_override_from_label(label);
-                                if w_id.is_some() {
-                                    let row_o = sqlx::query_as::<_, (String, bool, bool)>(
-                                        "SELECT username, is_admin, operator FROM usr where email = $1 AND \
-                                         workspace_id = $2 AND disabled = false",
-                                    )
-                                    .bind(&email)
-                                    .bind(&w_id.as_ref().unwrap())
-                                    .fetch_optional(&self.db)
-                                    .await
-                                    .unwrap_or(Some(("error".to_string(), false, false)));
-
-                                    match row_o {
-                                        Some((username, is_admin, is_operator)) => {
-                                            let groups = get_groups_for_user(
-                                                &w_id.as_ref().unwrap(),
-                                                &username,
-                                                &email,
-                                                &self.db,
-                                            )
-                                            .await
-                                            .ok()
-                                            .unwrap_or_default();
-
-                                            let folders = get_folders_for_user(
-                                                &w_id.unwrap(),
-                                                &username,
-                                                &groups,
-                                                &self.db,
-                                            )
-                                            .await
-                                            .ok()
-                                            .unwrap_or_default();
-                                            Some(ApiAuthed {
-                                                email,
-                                                username,
-                                                is_admin: is_admin || super_admin,
-                                                is_operator,
-                                                groups,
-                                                folders,
-                                                scopes,
-                                                username_override,
-                                            })
-                                        }
-                                        None if super_admin => Some(ApiAuthed {
-                                            email: email.clone(),
-                                            username: email,
-                                            is_admin: super_admin,
-                                            is_operator: false,
-                                            groups: vec![],
-                                            folders: vec![],
-                                            scopes,
-                                            username_override,
-                                        }),
-                                        None => None,
-                                    }
-                                } else {
-                                    Some(ApiAuthed {
-                                        email: email.to_string(),
-                                        username: email,
-                                        is_admin: super_admin,
-                                        is_operator: true,
-                                        groups: Vec::new(),
-                                        folders: Vec::new(),
-                                        scopes,
-                                        username_override,
-                                    })
-                                }
-                            }
-                            _ => None,
-                        }
-                    };
-                    if let Some(authed) = authed_o.as_ref() {
-                        self.cache.insert(
-                            key,
-                            ExpiringAuthCache {
-                                authed: authed.clone(),
-                                expiry: chrono::Utc::now()
-                                    + chrono::Duration::try_seconds(120).unwrap(),
-                            },
-                        );
-                    }
-                    authed_o
-                } else if self
-                    .superadmin_secret
-                    .as_ref()
-                    .map(|x| x == token)
-                    .unwrap_or(false)
-                {
-                    Some(ApiAuthed {
-                        email: SUPERADMIN_SECRET_EMAIL.to_string(),
-                        username: "superadmin_secret".to_string(),
-                        is_admin: true,
-                        is_operator: false,
-                        groups: Vec::new(),
-                        folders: Vec::new(),
-                        scopes: None,
-                        username_override: None,
-                    })
-                } else {
-                    None
-                }
-            }
-        }
-    }
-}
-
-async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
-    let auth_header = parts
-        .headers
-        .get(http::header::AUTHORIZATION)
-        .and_then(|value| value.to_str().ok())
-        .and_then(|s| s.strip_prefix("Bearer "));
-
-    let from_cookie = match auth_header {
-        Some(x) => Some(x.to_owned()),
-        None => Extension::<Cookies>::from_request_parts(parts, state)
-            .await
-            .ok()
-            .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())),
-    };
-
-    #[derive(Deserialize)]
-    struct Token {
-        token: Option<String>,
-    }
-    match from_cookie {
-        Some(token) => Some(token),
-        None => Query::<Token>::from_request_parts(parts, state)
-            .await
-            .ok()
-            .and_then(|token| token.token.clone()),
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct Tokened {
-    pub token: String,
-}
-
-pub struct OptTokened {
-    pub token: Option<String>,
-}
-
-struct BruteForceCounter {
-    counter: AtomicU64,
-    last_reset: AtomicI64,
-}
-
-lazy_static! {
-    static ref BRUTE_FORCE_COUNTER: BruteForceCounter =
-        BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) };
-}
-
-impl BruteForceCounter {
-    async fn increment(&self) {
-        let now = time::OffsetDateTime::now_utc().unix_timestamp();
-        if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 {
-            tracing::error!(
-                "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds"
-            );
-            tokio::time::sleep(std::time::Duration::from_secs(2)).await;
-        }
-        if now - self.last_reset.load(Ordering::Relaxed) > 60 {
-            self.counter.store(0, Ordering::Relaxed);
-            self.last_reset.store(now, Ordering::Relaxed);
-        }
-    }
-}
-
-#[async_trait]
-impl<S> FromRequestParts<S> for Tokened
-where
-    S: Send + Sync,
-{
-    type Rejection = (StatusCode, String);
-
-    async fn from_request_parts(
-        parts: &mut Parts,
-        state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
-        if parts.method == http::Method::OPTIONS {
-            return Ok(Tokened { token: "".to_string() });
-        };
-        let already_tokened = parts.extensions.get::<Tokened>();
-        if let Some(tokened) = already_tokened {
-            Ok(tokened.clone())
-        } else {
-            let token_o = extract_token(parts, state).await;
-            if let Some(token) = token_o {
-                let tokened = Self { token };
-                parts.extensions.insert(tokened.clone());
-                Ok(tokened)
-            } else {
-                BRUTE_FORCE_COUNTER.increment().await;
-                Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
-            }
-        }
-    }
-}
-
-#[async_trait]
-impl<S> FromRequestParts<S> for OptTokened
-where
-    S: Send + Sync,
-{
-    type Rejection = (StatusCode, String);
-
-    async fn from_request_parts(
-        parts: &mut Parts,
-        state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
-        if parts.method == http::Method::OPTIONS {
-            return Ok(OptTokened { token: None });
-        };
-        let already_tokened = parts.extensions.get::<Tokened>();
-        if let Some(tokened) = already_tokened {
-            Ok(OptTokened { token: Some(tokened.token.clone()) })
-        } else {
-            let token_o = extract_token(parts, state).await;
-            Ok(OptTokened { token: token_o })
-        }
-    }
-}
-
 pub async fn maybe_refresh_folders(
     path: &str,
     w_id: &str,
@@ -628,94 +150,6 @@ pub async fn maybe_refresh_folders(
     }
 }
 
-#[async_trait]
-impl<S> FromRequestParts<S> for ApiAuthed
-where
-    S: Send + Sync,
-{
-    type Rejection = (StatusCode, String);
-
-    async fn from_request_parts(
-        parts: &mut Parts,
-        state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
-        if parts.method == http::Method::OPTIONS {
-            return Ok(ApiAuthed {
-                email: "".to_owned(),
-                username: "".to_owned(),
-                is_admin: false,
-                is_operator: false,
-                groups: Vec::new(),
-                folders: Vec::new(),
-                scopes: None,
-                username_override: None,
-            });
-        };
-        let already_authed = parts.extensions.get::<ApiAuthed>();
-        if let Some(authed) = already_authed {
-            Ok(authed.clone())
-        } else {
-            let already_tokened = parts.extensions.get::<Tokened>();
-            let token_o = if let Some(token) = already_tokened {
-                Some(token.token.clone())
-            } else {
-                extract_token(parts, state).await
-            };
-            let original_uri = OriginalUri::from_request_parts(parts, state)
-                .await
-                .ok()
-                .map(|x| x.0)
-                .unwrap_or_default();
-            let path_vec: Vec<&str> = original_uri.path().split("/").collect();
-
-            let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" {
-                Some(path_vec[3].to_owned())
-            } else {
-                if path_vec.len() >= 5
-                    && path_vec[0] == ""
-                    && path_vec[2] == "srch"
-                    && path_vec[3] == "w"
-                {
-                    Some(path_vec[4].to_string())
-                } else {
-                    None
-                }
-            };
-            if let Some(token) = token_o {
-                if let Ok(Extension(cache)) =
-                    Extension::<Arc<AuthCache>>::from_request_parts(parts, state).await
-                {
-                    if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
-                        parts.extensions.insert(authed.clone());
-                        if authed.scopes.as_ref().is_some_and(|scopes| {
-                            scopes
-                                .iter()
-                                .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
-                        }) && (path_vec.len() < 3
-                            || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
-                        {
-                            BRUTE_FORCE_COUNTER.increment().await;
-                            return Err((
-                                StatusCode::UNAUTHORIZED,
-                                format!("Unauthorized scoped token: {:?}", authed.scopes),
-                            ));
-                        }
-                        Span::current().record("username", &authed.username.as_str());
-                        Span::current().record("email", &authed.email);
-
-                        if let Some(workspace_id) = workspace_id {
-                            Span::current().record("workspace_id", &workspace_id);
-                        }
-                        return Ok(authed);
-                    }
-                }
-            }
-            BRUTE_FORCE_COUNTER.increment().await;
-            Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
-        }
-    }
-}
-
 pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
 where
     F: FnOnce() -> String,
@@ -768,6 +202,7 @@ where
     }
 }
 
+#[allow(unused)]
 pub async fn fetch_api_authed(
     username: String,
     email: String,
@@ -779,6 +214,7 @@ pub async fn fetch_api_authed(
     fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await
 }
 
+#[allow(unused)]
 pub async fn fetch_api_authed_from_permissioned_as(
     permissioned_as: String,
     email: String,
@@ -2217,13 +1653,30 @@ async fn login(
     }
 }
 
+#[derive(Deserialize)]
+struct RefreshTokenQuery {
+    if_expiring_in_less_than_s: Option<i32>,
+}
 async fn refresh_token(
     Extension(db): Extension<DB>,
+    Query(query): Query<RefreshTokenQuery>,
+    Tokened { token }: Tokened,
     authed: ApiAuthed,
     cookies: Cookies,
 ) -> Result<String> {
     let mut tx = db.begin().await?;
 
+    if let Some(thresh_s) = query.if_expiring_in_less_than_s {
+        let not_expired = sqlx::query_scalar!("SELECT true FROM token WHERE token = $1 and expiration IS NOT NULL and expiration > now() + $2::int * '1 sec'::interval", &token, thresh_s)
+            .fetch_optional(&db)
+            .await?
+            .flatten()
+            .unwrap_or(false);
+        if not_expired {
+            return Ok("token expiry is far enough".to_string());
+        }
+    }
+
     let super_admin = sqlx::query_scalar!(
         "SELECT super_admin FROM password WHERE email = $1",
         &authed.email
@@ -2494,7 +1947,7 @@ async fn get_all_runnables(
     Extension(db): Extension<UserDB>,
     authed: ApiAuthed,
     Tokened { token }: Tokened,
-    Extension(cache): Extension<Arc<AuthCache>>,
+    Extension(cache): Extension<Arc<crate::auth::AuthCache>>,
 ) -> JsonResult<Vec<Runnable>> {
     let mut tx = db.clone().begin(&authed).await?;
     let mut runnables = Vec::new();

backend/windmill-api/src/utils.rs

@@ -10,6 +10,7 @@ use axum::{body::Body, response::Response};
 use regex::Regex;
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
+#[cfg(feature = "enterprise")]
 use windmill_common::worker::CLOUD_HOSTED;
 use windmill_common::{
     auth::{is_devops_email, is_super_admin_email},

backend/windmill-api/src/variables.rs

@@ -8,7 +8,6 @@
 
 use crate::{
     db::{ApiAuthed, DB},
-    oauth2_ee::_refresh_token,
     users::{maybe_refresh_folders, require_owner_of_path},
     webhook_util::{WebhookMessage, WebhookShared},
 };
@@ -183,10 +182,21 @@ async fn get_variable(
         let value = variable.value.unwrap_or_else(|| "".to_string());
         ListableVariable {
             value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-                Some(
-                    _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap(), &db)
+                #[cfg(feature = "oauth2")]
+                {
+                    Some(
+                        crate::oauth2_ee::_refresh_token(
+                            tx,
+                            &variable.path,
+                            &w_id,
+                            variable.account.unwrap(),
+                            &db,
+                        )
                         .await?,
-                )
+                    )
+                }
+                #[cfg(not(feature = "oauth2"))]
+                return Err(Error::InternalErr("Require oauth2 feature".to_string()));
             } else if !value.is_empty() && decrypt_secret {
                 let _ = tx.commit().await;
                 let mc = build_crypt(&db, &w_id).await?;
@@ -640,7 +650,19 @@ pub async fn get_value_internal<'c>(
         .await?;
         let value = variable.value;
         if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-            _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap(), db).await?
+            #[cfg(feature = "oauth2")]
+            {
+                crate::oauth2_ee::_refresh_token(
+                    tx,
+                    &variable.path,
+                    &w_id,
+                    variable.account.unwrap(),
+                    db,
+                )
+                .await?
+            }
+            #[cfg(not(feature = "oauth2"))]
+            return Err(Error::InternalErr("Require oauth2 feature".to_string()));
         } else if !value.is_empty() {
             tx.commit().await?;
             let mc = build_crypt(&db, &w_id).await?;

backend/windmill-api/src/webhook_util.rs

@@ -8,7 +8,7 @@ use tokio::{select, sync::mpsc};
 use windmill_common::METRICS_ENABLED;
 
 use crate::db::DB;
-use crate::oauth2_ee::InstanceEvent;
+use windmill_common::oauth2::InstanceEvent;
 
 #[cfg(feature = "prometheus")]
 lazy_static::lazy_static! {

backend/windmill-api/src/websocket_triggers.rs

@@ -837,6 +837,7 @@ async fn listen_to_websocket(
                                 if let Err(err) = send_initial_messages(&ws_trigger, writer, &db).await {
                                     disable_with_error(&db, &ws_trigger, format!("Error sending initial messages: {:?}", err)).await;
                                 } else {
+                                    tracing::debug!("Initial messages sent successfully to websocket {}", url);
                                     // if initial messages sent successfully, wait forever
                                     futures::future::pending::<()>().await;
                                 }
@@ -860,6 +861,7 @@ async fn listen_to_websocket(
                                                     Ok(msg) => {
                                                         match msg {
                                                             tokio_tungstenite::tungstenite::Message::Text(text) => {
+                                                                tracing::debug!("Received text message from websocket {}: {}", url, text);
                                                                 let mut should_handle = true;
                                                                 for filter in &filters {
                                                                     match filter {
@@ -886,7 +888,9 @@ async fn listen_to_websocket(
                                                                     };
                                                                 }
                                                             },
-                                                            _ => {}
+                                                            a @ _ => {
+                                                                tracing::debug!("Received non text-message from websocket {}: {:?}", url, a);
+                                                            }
                                                         }
                                                     },
                                                     Err(err) => {
@@ -905,6 +909,7 @@ async fn listen_to_websocket(
                                             }
                                         },
                                         _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => {
+                                            tracing::debug!("Sending ping to websocket {}", url);
                                             if let None = update_ping(&db, &ws_trigger, None).await {
                                                 return;
                                             }

backend/windmill-api/src/workers.rs

@@ -185,4 +185,4 @@ async fn get_queue_counts(
     require_super_admin(&db, &authed.email).await?;
     let queue_counts = windmill_common::queue::get_queue_counts(&db).await;
     Ok(Json(queue_counts))
-}
+}

backend/windmill-api/src/workspaces_export.rs

@@ -0,0 +1,676 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use std::collections::HashMap;
+
+use crate::db::ApiAuthed;
+
+use crate::{
+    apps::AppWithLastVersion,
+    db::DB,
+    folders::Folder,
+    resources::{Resource, ResourceType},
+};
+
+use axum::{
+    extract::{Extension, Path, Query},
+    response::IntoResponse,
+};
+
+use http::HeaderName;
+use itertools::Itertools;
+
+use windmill_common::db::UserDB;
+use windmill_common::schedule::Schedule;
+use windmill_common::variables::build_crypt;
+
+use windmill_common::{
+    error::{to_anyhow, Error, Result},
+    flows::Flow,
+    scripts::{Schema, Script, ScriptLang},
+    variables::ExportableListableVariable,
+};
+
+use crate::variables::decrypt;
+use hyper::header;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tempfile::TempDir;
+use tokio::fs::File;
+use tokio_util::io::ReaderStream;
+
+#[derive(Serialize)]
+struct ScriptMetadata {
+    summary: String,
+    description: String,
+    schema: Option<Schema>,
+    lock: Option<String>,
+    kind: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    envs: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    concurrent_limit: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    concurrency_time_window_s: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    cache_ttl: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    dedicated_worker: Option<bool>,
+    #[serde(skip_serializing_if = "is_none_or_false")]
+    ws_error_handler_muted: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    priority: Option<i16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub timeout: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delete_after_use: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub restart_unless_cancelled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub no_main_func: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub codebase: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub concurrency_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_preprocessor: Option<bool>,
+}
+
+pub fn is_none_or_false(val: &Option<bool>) -> bool {
+    match val {
+        Some(val) => !val,
+        None => true,
+    }
+}
+
+enum ArchiveImpl {
+    #[cfg(feature = "zip")]
+    Zip(async_zip::tokio::write::ZipFileWriter<tokio::fs::File>),
+    Tar(tokio_tar::Builder<File>),
+}
+
+impl ArchiveImpl {
+    async fn write_to_archive(&mut self, content: &str, path: &str) -> Result<()> {
+        match self {
+            ArchiveImpl::Tar(t) => {
+                let bytes = content.as_bytes();
+                let mut header = tokio_tar::Header::new_gnu();
+                header.set_size(bytes.len() as u64);
+                header.set_mtime(0);
+                header.set_uid(0);
+                header.set_gid(0);
+                header.set_mode(0o777);
+                header.set_cksum();
+                t.append_data(&mut header, path, bytes).await?;
+            }
+            #[cfg(feature = "zip")]
+            ArchiveImpl::Zip(z) => {
+                let header =
+                    async_zip::ZipEntryBuilder::new(path.into(), async_zip::Compression::Deflate)
+                        .last_modification_date(Default::default())
+                        .unix_permissions(0o777)
+                        .build();
+                z.write_entry_whole(header, content.as_bytes())
+                    .await
+                    .map_err(to_anyhow)?;
+            }
+        }
+        Ok(())
+    }
+    async fn finish(self) -> Result<()> {
+        match self {
+            ArchiveImpl::Tar(t) => t.into_inner().await?,
+            #[cfg(feature = "zip")]
+            ArchiveImpl::Zip(z) => z.close().await.map_err(to_anyhow)?.into_inner(),
+        }
+        .sync_all()
+        .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(Deserialize)]
+pub(crate) struct ArchiveQueryParams {
+    archive_type: Option<String>,
+    plain_secret: Option<bool>,
+    plain_secrets: Option<bool>,
+    skip_secrets: Option<bool>,
+    skip_variables: Option<bool>,
+    skip_resources: Option<bool>,
+    include_schedules: Option<bool>,
+    include_users: Option<bool>,
+    include_groups: Option<bool>,
+    include_settings: Option<bool>,
+    include_key: Option<bool>,
+    default_ts: Option<String>,
+}
+
+#[inline]
+pub fn to_string_without_metadata<T>(
+    value: &T,
+    preserve_extra_perms: bool,
+    ignore_keys: Option<Vec<&str>>,
+) -> Result<String>
+where
+    T: ?Sized + Serialize,
+{
+    let mut value = serde_json::to_value(value).map_err(to_anyhow)?;
+    value
+        .as_object_mut()
+        .map(|obj| {
+            let keys = [
+                vec![
+                    "workspace_id",
+                    "path",
+                    "name",
+                    "versions",
+                    "id",
+                    "created_at",
+                    "updated_at",
+                    "created_by",
+                    "updated_by",
+                    "edited_at",
+                    "edited_by",
+                    "archived",
+                    "has_draft",
+                    "draft_only",
+                    "error",
+                ],
+                ignore_keys.unwrap_or(vec![]),
+            ]
+            .concat();
+
+            for key in keys {
+                if obj.contains_key(key) {
+                    obj.remove(key);
+                }
+            }
+
+            if let Some(o2) = obj.get_mut("policy").and_then(|x| x.as_object_mut()) {
+                o2.remove("on_behalf_of");
+                o2.remove("on_behalf_of_email");
+            }
+            if !preserve_extra_perms && obj.contains_key("extra_perms") {
+                obj.remove("extra_perms");
+            }
+
+            serde_json::to_string_pretty(&obj).ok()
+        })
+        .flatten()
+        .ok_or_else(|| Error::BadRequest("Impossible to serialize value".to_string()))
+}
+
+#[derive(Serialize)]
+struct SimplifiedUser {
+    username: String,
+    role: String,
+    disabled: bool,
+    email: String,
+}
+
+#[derive(Serialize)]
+struct SimplifiedGroup {
+    name: String,
+    summary: Option<String>,
+    members: Vec<String>,
+    admins: Vec<String>,
+}
+
+#[derive(Serialize)]
+struct SimplifiedSettings {
+    // slack_team_id: Option<String>,
+    // slack_name: Option<String>,
+    // slack_command_script: Option<String>,
+    // slack_email: Option<String>,
+    auto_invite_enabled: bool,
+    auto_invite_as: String,
+    auto_invite_mode: String,
+    webhook: Option<String>,
+    deploy_to: Option<String>,
+    error_handler: Option<String>,
+    error_handler_extra_args: Option<Value>,
+    error_handler_muted_on_cancel: bool,
+    ai_resource: Option<serde_json::Value>,
+    code_completion_enabled: bool,
+    large_file_storage: Option<Value>,
+    git_sync: Option<Value>,
+    default_app: Option<String>,
+    default_scripts: Option<Value>,
+    name: String,
+}
+
+pub(crate) async fn tarball_workspace(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Query(ArchiveQueryParams {
+        archive_type,
+        plain_secret,
+        plain_secrets,
+        skip_resources,
+        skip_secrets,
+        skip_variables,
+        include_schedules,
+        include_users,
+        include_groups,
+        include_settings,
+        include_key,
+        default_ts,
+    }): Query<ArchiveQueryParams>,
+) -> Result<([(HeaderName, String); 2], impl IntoResponse)> {
+    // require_admin(authed.is_admin, &authed.username)?;
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    let tmp_dir = TempDir::new_in("/tmp/windmill/")?;
+
+    let name = match archive_type.as_deref() {
+        Some("tar") | None => Ok(format!("windmill-{w_id}.tar")),
+        Some("zip") => Ok(format!("windmill-{w_id}.zip")),
+        Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))),
+    }?;
+    let file_path = tmp_dir.path().join(&name);
+    let mut archive = match archive_type.as_deref() {
+        Some("tar") | None => {
+            let file = File::create(&file_path).await?;
+            Ok(ArchiveImpl::Tar(tokio_tar::Builder::new(file)))
+        }
+        #[cfg(feature = "zip")]
+        Some("zip") => {
+            let file = tokio::fs::File::create(&file_path).await?;
+            Ok(ArchiveImpl::Zip(
+                async_zip::tokio::write::ZipFileWriter::with_tokio(file),
+            ))
+        }
+        Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))),
+    }?;
+    {
+        let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1")
+            .bind(&w_id)
+            .fetch_all(&mut *tx)
+            .await?;
+
+        for folder in folders {
+            archive
+                .write_to_archive(
+                    &to_string_without_metadata(&folder, true, None).unwrap(),
+                    &format!("f/{}/folder.meta.json", folder.name),
+                )
+                .await?;
+        }
+    }
+
+    {
+        let scripts = sqlx::query_as::<_, Script>(
+            "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
+            AND created_at = (select max(created_at) from script where path = o.path AND \
+             workspace_id = $1)",
+        )
+        .bind(&w_id)
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for script in scripts {
+            let ext = match script.language {
+                ScriptLang::Python3 => "py",
+                ScriptLang::Deno => {
+                    if default_ts.as_ref().is_some_and(|x| x == "bun") {
+                        "deno.ts"
+                    } else {
+                        "ts"
+                    }
+                }
+                ScriptLang::Go => "go",
+                ScriptLang::Bash => "sh",
+                ScriptLang::Powershell => "ps1",
+                ScriptLang::Postgresql => "pg.sql",
+                ScriptLang::Mysql => "my.sql",
+                ScriptLang::Bigquery => "bq.sql",
+                ScriptLang::Snowflake => "sf.sql",
+                ScriptLang::Mssql => "ms.sql",
+                ScriptLang::Graphql => "gql",
+                ScriptLang::Nativets => "fetch.ts",
+                ScriptLang::Bun | ScriptLang::Bunnative => {
+                    if default_ts.as_ref().is_some_and(|x| x == "bun") {
+                        "ts"
+                    } else {
+                        "bun.ts"
+                    }
+                }
+                ScriptLang::Php => "php",
+                ScriptLang::Rust => "rs",
+                ScriptLang::Ansible => "playbook.yml",
+                ScriptLang::CSharp => "cs",
+            };
+            archive
+                .write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
+                .await?;
+
+            let metadata = ScriptMetadata {
+                summary: script.summary,
+                description: script.description,
+                schema: script.schema,
+                kind: script.kind.to_string(),
+                lock: script.lock,
+                envs: script.envs,
+                concurrent_limit: script.concurrent_limit,
+                concurrency_time_window_s: script.concurrency_time_window_s,
+                cache_ttl: script.cache_ttl,
+                dedicated_worker: script.dedicated_worker,
+                ws_error_handler_muted: script.ws_error_handler_muted,
+                priority: script.priority,
+                tag: script.tag,
+                timeout: script.timeout,
+                delete_after_use: script.delete_after_use,
+                restart_unless_cancelled: script.restart_unless_cancelled,
+                visible_to_runner_only: script.visible_to_runner_only,
+                no_main_func: script.no_main_func,
+                codebase: script.codebase,
+                concurrency_key: script.concurrency_key,
+                has_preprocessor: script.has_preprocessor,
+            };
+            let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
+            archive
+                .write_to_archive(&metadata_str, &format!("{}.script.json", script.path))
+                .await?;
+        }
+    }
+
+    if !skip_resources.unwrap_or(false) {
+        let resources = sqlx::query_as!(
+            Resource,
+            "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
+            &w_id
+        )
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for resource in resources {
+            let resource_str = &to_string_without_metadata(&resource, false, None).unwrap();
+            archive
+                .write_to_archive(&resource_str, &format!("{}.resource.json", resource.path))
+                .await?;
+        }
+    }
+
+    if !skip_resources.unwrap_or(false) {
+        let resource_types = sqlx::query_as!(
+            ResourceType,
+            "SELECT * FROM resource_type WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for resource_type in resource_types {
+            let resource_str = &to_string_without_metadata(&resource_type, false, None).unwrap();
+            archive
+                .write_to_archive(
+                    &resource_str,
+                    &format!("{}.resource-type.json", resource_type.name),
+                )
+                .await?;
+        }
+    }
+
+    {
+        let flows = sqlx::query_as::<_, Flow>(
+            "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by
+            FROM flow
+            LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
+            WHERE flow.workspace_id = $1 AND flow.archived = false",
+        )
+        .bind(&w_id)
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for flow in flows {
+            let flow_str = &to_string_without_metadata(&flow, false, None).unwrap();
+            archive
+                .write_to_archive(&flow_str, &format!("{}.flow.json", flow.path))
+                .await?;
+        }
+    }
+
+    if !skip_variables.unwrap_or(false) {
+        let variables =
+            sqlx::query_as::<_, ExportableListableVariable>(if !skip_secrets.unwrap_or(false) {
+                "SELECT * FROM variable WHERE workspace_id = $1 AND expires_at IS NULL"
+            } else {
+                "SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL"
+            })
+            .bind(&w_id)
+            .fetch_all(&mut *tx)
+            .await?;
+
+        let mc = build_crypt(&db, &w_id).await?;
+
+        for mut var in variables {
+            if plain_secret.or(plain_secrets).unwrap_or(false)
+                && var.value.is_some()
+                && var.is_secret
+            {
+                var.value = Some(decrypt(&mc, var.value.unwrap())?);
+            }
+            let var_str = &to_string_without_metadata(&var, false, None).unwrap();
+            archive
+                .write_to_archive(&var_str, &format!("{}.variable.json", var.path))
+                .await?;
+        }
+    }
+
+    {
+        let apps = sqlx::query_as::<_, AppWithLastVersion>(
+            "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
+            app.extra_perms, app_version.value, 
+            app_version.created_at, app_version.created_by from app, app_version 
+            WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
+        )
+        .bind(&w_id)
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for app in apps {
+            let app_str = &to_string_without_metadata(&app, false, None).unwrap();
+            archive
+                .write_to_archive(&app_str, &format!("{}.app.json", app.path))
+                .await?;
+        }
+    }
+
+    if include_schedules.unwrap_or(false) {
+        let schedules = sqlx::query_as::<_, Schedule>(
+            "SELECT * FROM schedule
+            WHERE workspace_id = $1",
+        )
+        .bind(&w_id)
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for schedule in schedules {
+            let app_str = &to_string_without_metadata(&schedule, false, None).unwrap();
+            archive
+                .write_to_archive(&app_str, &format!("{}.schedule.json", schedule.path))
+                .await?;
+        }
+    }
+
+    if include_users.unwrap_or(false) {
+        let users = sqlx::query!(
+            "SELECT * FROM usr
+            WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for user in users {
+            let user = SimplifiedUser {
+                username: user.username,
+                role: if user.is_admin {
+                    "admin".to_string()
+                } else if user.operator {
+                    "operator".to_string()
+                } else {
+                    "developer".to_string()
+                },
+                disabled: user.disabled,
+                email: user.email,
+            };
+            let user_str = &to_string_without_metadata(
+                &user,
+                false,
+                Some(vec!["is_admin", "operator", "email"]),
+            )
+            .unwrap();
+            archive
+                .write_to_archive(&user_str, &format!("users/{}.user.json", user.email))
+                .await?;
+        }
+    }
+
+    if include_groups.unwrap_or(false) {
+        let groups = sqlx::query!(
+            r#"SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members 
+            FROM usr u
+            JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id
+            RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_
+            WHERE g_.workspace_id = $1 AND g_.name != 'all'
+            GROUP BY g_.workspace_id, name, summary, extra_perms"#,
+            &w_id
+        )
+        .fetch_all(&mut *tx)
+        .await?;
+
+        for group in groups {
+            let extra_perms: HashMap<String, bool> = serde_json::from_value(group.extra_perms)
+                .map_err(|e| {
+                    Error::InternalErr(format!(
+                        "Error parsing extra_perms for group {}: {}",
+                        group.name, e
+                    ))
+                })?;
+            tracing::info!("{:?}", extra_perms);
+            let members = group.members.unwrap_or(vec![]);
+            let admins: Vec<String> = extra_perms
+                .iter()
+                .filter_map(|(k, v)| {
+                    // only consider extra_perms that concern actual members of the group
+                    if members.contains(&k[2..].to_string()) && *v {
+                        Some(k.clone())
+                    } else {
+                        None
+                    }
+                })
+                .sorted()
+                .collect();
+            let group = SimplifiedGroup {
+                name: group.name,
+                summary: group.summary,
+                members: members
+                    .iter()
+                    .filter_map(|x| {
+                        // remove members that are also admins as they are already in the admins list
+                        let full_name = format!("u/{}", x);
+                        if !admins.contains(&full_name) {
+                            Some(full_name)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect(),
+                admins,
+            };
+
+            let group_str = &to_string_without_metadata(&group, true, None).unwrap();
+            archive
+                .write_to_archive(&group_str, &format!("groups/{}.group.json", group.name))
+                .await?;
+        }
+    }
+
+    if include_settings.unwrap_or(false) {
+        let settings = sqlx::query_as!(
+            SimplifiedSettings,
+            r#"SELECT
+                -- slack_team_id, 
+                -- slack_name, 
+                -- slack_command_script, 
+                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,
+                auto_invite_domain IS NOT NULL AS "auto_invite_enabled!",
+                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!", 
+                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!", 
+                webhook, 
+                deploy_to, 
+                error_handler, 
+                ai_resource, 
+                code_completion_enabled, 
+                error_handler_extra_args, 
+                error_handler_muted_on_cancel, 
+                large_file_storage, 
+                git_sync,
+                default_app,
+                default_scripts,
+                workspace.name
+            FROM workspace_settings
+            LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id
+            WHERE workspace_id = $1"#,
+            &w_id
+        ).fetch_one(&mut *tx).await?;
+
+        let settings_str = serde_json::to_value(settings)
+            .map(|v| serde_json::to_string_pretty(&v).ok())
+            .ok()
+            .flatten()
+            .ok_or_else(|| Error::InternalErr("Error serializing settings".to_string()))?;
+
+        archive
+            .write_to_archive(&settings_str, "settings.json")
+            .await?;
+    }
+
+    if include_key.unwrap_or(false) {
+        let key = sqlx::query_scalar!(
+            "SELECT key FROM workspace_key WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_one(&mut *tx)
+        .await?;
+
+        let key_json = serde_json::to_value(key)
+            .map(|v| serde_json::to_string_pretty(&v).ok())
+            .ok()
+            .flatten()
+            .ok_or_else(|| Error::InternalErr("Error serializing enryption key".to_string()))?;
+        archive
+            .write_to_archive(&key_json, "encryption_key.json")
+            .await?;
+    }
+
+    archive.finish().await?;
+
+    let file = tokio::fs::File::open(&file_path).await?;
+
+    let stream = ReaderStream::new(file);
+    let body = axum::body::Body::from_stream(stream);
+
+    let headers = [
+        (header::CONTENT_TYPE, "application/x-tar".to_string()),
+        (
+            header::CONTENT_DISPOSITION,
+            format!("attachment; filename=\"{name}\""),
+        ),
+    ];
+    Ok((headers, body))
+}

backend/windmill-api/src/workspaces_extra.rs

@@ -0,0 +1,506 @@
+use crate::db::ApiAuthed;
+
+use crate::workspaces::CREATE_WORKSPACE_REQUIRE_SUPERADMIN;
+use crate::{db::DB, utils::require_super_admin};
+
+use axum::{
+    extract::{Extension, Path},
+    Json,
+};
+
+use windmill_audit::audit_ee::audit_log;
+use windmill_audit::ActionKind;
+
+use windmill_common::worker::CLOUD_HOSTED;
+
+use windmill_common::{
+    error::{Error, Result},
+    utils::require_admin,
+};
+
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+pub(crate) struct ChangeWorkspaceId {
+    new_id: String,
+    new_name: String,
+}
+
+pub(crate) async fn change_workspace_id(
+    authed: ApiAuthed,
+    Path(old_id): Path<String>,
+    Extension(db): Extension<DB>,
+    Json(rw): Json<ChangeWorkspaceId>,
+) -> Result<String> {
+    if *CLOUD_HOSTED {
+        return Err(Error::BadRequest(
+            "This feature is not available on the cloud".to_string(),
+        ));
+    }
+
+    if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN {
+        require_super_admin(&db, &authed.email).await?;
+    } else {
+        require_admin(authed.is_admin, &authed.username)?;
+    }
+
+    let mut tx = db.begin().await?;
+
+    let workspace_conflict = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)",
+        &rw.new_id,
+    )
+    .fetch_one(&mut *tx)
+    .await?
+    .unwrap_or(false);
+
+    if workspace_conflict {
+        return Err(Error::BadRequest(format!(
+            "workspace id {} already used",
+            &rw.new_id
+        )));
+    }
+
+    // duplicate workspace with new id name
+    sqlx::query!(
+        "INSERT INTO workspace SELECT $1, $2, owner, deleted, premium FROM workspace WHERE id = $3",
+        &rw.new_id,
+        &rw.new_name,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE account SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE app SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE audit SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE capture SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE completed_job SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE dependency_map SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE deployment_metadata SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE draft SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE favorite SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "INSERT INTO flow 
+            (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at) 
+        SELECT $1, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at
+            FROM flow WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE flow_version SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE flow_node SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!("DELETE FROM flow WHERE workspace_id = $1", &old_id)
+        .execute(&mut *tx)
+        .await?;
+
+    // have to duplicate group_ with new workspace id because of foreign key constraint
+    sqlx::query!(
+        "INSERT INTO group_ SELECT $1, name, summary, extra_perms FROM group_ WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE usr_to_group SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    // then delete old group_
+    sqlx::query!("DELETE FROM group_ WHERE workspace_id = $1", &old_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!(
+        "UPDATE folder SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE input SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE job_logs SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE job_stats SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE queue SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE job SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE raw_app SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE resource SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE resource_type SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE schedule SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE script SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE token SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE usage SET id = $1 WHERE id = $2 AND is_workspace = true",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE usr SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE variable SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE workspace_env SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE workspace_invite SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE workspace_key SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!(
+        "UPDATE workspace_settings SET workspace_id = $1 WHERE workspace_id = $2",
+        &rw.new_id,
+        &old_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    // delete old workspace
+    sqlx::query!("DELETE FROM workspace WHERE id = $1", &old_id)
+        .execute(&mut *tx)
+        .await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "workspace.change_workspace_id",
+        ActionKind::Update,
+        &rw.new_id,
+        Some(&authed.email),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(format!(
+        "updated workspace from {} to {}",
+        &old_id, &rw.new_id
+    ))
+}
+
+pub(crate) async fn delete_workspace(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    authed: ApiAuthed,
+) -> Result<String> {
+    let w_id = match w_id.as_str() {
+        "starter" => Err(Error::BadRequest(
+            "starter workspace cannot be deleted".to_string(),
+        )),
+        "admins" => Err(Error::BadRequest(
+            "admins workspace cannot be deleted".to_string(),
+        )),
+        _ => Ok(w_id),
+    }?;
+    let mut tx = db.begin().await?;
+    require_super_admin(&db, &authed.email).await?;
+
+    sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM queue WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM capture WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM draft WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM script WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM flow WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM app WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM raw_app WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM input WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM variable WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM resource WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM schedule WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM completed_job WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM job_stats WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!(
+        "DELETE FROM deployment_metadata WHERE workspace_id = $1",
+        &w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!("DELETE FROM usr WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM resource_type WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!(
+        "DELETE FROM workspace_invite WHERE workspace_id = $1",
+        &w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!("DELETE FROM usr_to_group WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM group_ WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM folder WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM account WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM workspace_key WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!(
+        "DELETE FROM workspace_settings WHERE workspace_id = $1",
+        &w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "workspaces.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(&authed.email),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+
+    Ok(format!("Deleted workspace {}", &w_id))
+}

backend/windmill-common/Cargo.toml

@@ -9,12 +9,13 @@ default = []
 enterprise = []
 jemalloc = ["dep:tikv-jemalloc-ctl"]
 prometheus = ["dep:prometheus"]
-flamegraph = ["dep:tracing-flame"]
 loki = ["dep:tracing-loki"]
 benchmark = []
 parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
 otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", 
     "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
+smtp = ["dep:mail-send"]
+scoped_cache = []
 
 [lib]
 name = "windmill_common"
@@ -40,7 +41,6 @@ tokio = { workspace = true }
 reqwest = { workspace = true }
 tracing-subscriber = { workspace = true }
 lazy_static.workspace = true
-tracing-flame = { version = "^0", optional = true }
 tracing-appender.workspace = true
 gethostname.workspace = true
 itertools.workspace = true
@@ -55,7 +55,7 @@ aws-config = { workspace = true, optional = true }
 aws-sdk-sts = { workspace = true, optional = true }
 indexmap.workspace = true
 bytes.workspace = true
-mail-send.workspace = true
+mail-send = { workspace = true, optional = true }
 futures-core.workspace = true
 async-stream.workspace = true
 const_format.workspace = true
@@ -65,6 +65,8 @@ semver.workspace = true
 croner = "2.0.6"
 quick_cache.workspace = true
 pin-project-lite.workspace = true
+futures.workspace = true
+tempfile.workspace = true
 
 opentelemetry-semantic-conventions = { workspace = true, optional = true }
 opentelemetry-otlp = { workspace = true, optional = true }

backend/windmill-common/src/apps.rs

@@ -13,12 +13,6 @@ use serde::{Deserialize, Serialize};
 #[serde(transparent)]
 pub struct AppScriptId(pub i64);
 
-impl Into<u64> for AppScriptId {
-    fn into(self) -> u64 {
-        self.0 as u64
-    }
-}
-
 #[derive(Deserialize)]
 pub struct ListAppQuery {
     pub starred_only: Option<bool>,

backend/windmill-common/src/error.rs

@@ -6,6 +6,8 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::panic::Location;
+
 use axum::body::Body;
 use axum::response::Response;
 use axum::{response::IntoResponse, response::Json};
@@ -48,6 +50,8 @@ pub enum Error {
     QuotaExceeded(String),
     #[error("Internal: {0}")]
     InternalErr(String),
+    #[error("Internal: {0}: {1}")]
+    InternalErrAt(&'static Location<'static>, String),
     #[error("Hexadecimal decoding error: {0}")]
     HexErr(#[from] hex::FromHexError),
     #[error("Migrating database: {0}")]
@@ -77,6 +81,17 @@ impl Error {
     pub fn dbg(&self) -> String {
         format!("{:?}", self)
     }
+
+    pub fn relocate_internal(self, loc: &'static Location<'static>) -> Self {
+        match self {
+            Self::InternalErr(s) | Self::InternalErrAt(_, s) => Self::InternalErrAt(loc, s),
+            _ => self,
+        }
+    }
+}
+
+pub fn relocate_internal(loc: &'static Location<'static>) -> impl FnOnce(Error) -> Error {
+    move |e| e.relocate_internal(loc)
 }
 
 pub fn to_anyhow<T: 'static + std::error::Error + Send + Sync>(e: T) -> anyhow::Error {
@@ -92,10 +107,9 @@ impl IntoResponse for Error {
             Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND,
             Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
             Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
-            Self::SqlErr(_)
-            | Self::BadRequest(_)
-            | Self::AiError(_)
-            | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST,
+            Self::SqlErr(_) | Self::BadRequest(_) | Self::AiError(_) | Self::QuotaExceeded(_) => {
+                axum::http::StatusCode::BAD_REQUEST
+            }
             _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
         };
 

backend/windmill-common/src/flows.rs

@@ -407,12 +407,6 @@ pub enum InputTransform {
 #[serde(transparent)]
 pub struct FlowNodeId(pub i64);
 
-impl Into<u64> for FlowNodeId {
-    fn into(self) -> u64 {
-        self.0 as u64
-    }
-}
-
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Branch {
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -784,7 +778,9 @@ pub async fn resolve_module(
             let (lock, content) = if !with_code {
                 (Some("...".to_string()), "...".to_string())
             } else {
-                cache::flow::fetch_script(e, id).await?
+                cache::flow::fetch_script(e, id)
+                    .await
+                    .map(|data| (data.lock.clone(), data.code.clone()))?
             };
             val = RawScript {
                 input_transforms,
@@ -844,7 +840,7 @@ pub async fn resolve_modules(
     if let Some(id) = modules_node {
         *modules = cache::flow::fetch_flow(e, id)
             .await
-            .map(|flow| flow.modules)?;
+            .map(|data| data.value().modules.clone())?;
     }
     for module in modules.iter_mut() {
         Box::pin(resolve_module(

backend/windmill-common/src/jobs.rs

@@ -45,6 +45,15 @@ pub enum JobKind {
     AppScript,
 }
 
+impl JobKind {
+    pub fn is_flow(&self) -> bool {
+        matches!(
+            self,
+            JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode
+        )
+    }
+}
+
 #[derive(sqlx::FromRow, Debug, Serialize, Clone)]
 pub struct QueuedJob {
     pub workspace_id: String,
@@ -116,10 +125,7 @@ impl QueuedJob {
             .unwrap_or("tmp/main")
     }
     pub fn is_flow(&self) -> bool {
-        matches!(
-            self.job_kind,
-            JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode
-        )
+        self.job_kind.is_flow()
     }
 
     pub fn full_path_with_workspace(&self) -> String {
@@ -275,10 +281,11 @@ pub enum JobPayload {
         concurrency_time_window_s: Option<i32>,
         cache_ttl: Option<i32>,
         dedicated_worker: Option<bool>,
+        path: String,
     },
     FlowNode {
         id: FlowNodeId, // flow_node(id).
-        path: String, // flow node inner path (e.g. `outer/branchall-42`).
+        path: String,   // flow node inner path (e.g. `outer/branchall-42`).
     },
     AppScript {
         id: AppScriptId, // app_script(id).

backend/windmill-common/src/oauth2.rs

@@ -9,6 +9,7 @@
 use std::sync::atomic::AtomicBool;
 
 use hmac::Hmac;
+use serde::Serialize;
 use sha2::Sha256;
 
 pub type HmacSha256 = Hmac<Sha256>;
@@ -25,3 +26,15 @@ lazy_static::lazy_static! {
         .unwrap_or(false));
 
 }
+
+#[derive(Serialize)]
+#[serde(tag = "type")]
+pub enum InstanceEvent {
+    UserSignupOAuth { email: String },
+    UserAdded { email: String },
+    // UserDeleted { email: String },
+    // UserDeletedWorkspace { workspace: String, email: String },
+    UserAddedWorkspace { workspace: String, email: String },
+    UserInvitedWorkspace { workspace: String, email: String },
+    UserJoinedWorkspace { workspace: String, email: String, username: String },
+}

backend/windmill-common/src/scripts.rs

@@ -24,7 +24,7 @@ use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
 
 use crate::utils::StripPath;
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq, sqlx::Type)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]
 #[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
 #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
 pub enum ScriptLang {
@@ -77,12 +77,6 @@ impl ScriptLang {
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
 
-impl Into<u64> for ScriptHash {
-    fn into(self) -> u64 {
-        self.0 as u64
-    }
-}
-
 #[derive(PartialEq, sqlx::Type)]
 #[sqlx(transparent, no_pg_array)]
 pub struct ScriptHashes(pub Vec<i64>);

backend/windmill-common/src/tracing_init.rs

@@ -149,22 +149,6 @@ pub fn initialize_tracing(
     (_guard, meter_provider)
 }
 
-#[cfg(feature = "flamegraph")]
-use tracing_flame::FlameLayer;
-
-#[cfg(feature = "flamegraph")]
-pub fn setup_flamegraph() -> impl Drop {
-    // let fmt_layer = Layer::default();
-
-    let (flame_layer, _guard) = FlameLayer::with_file("./tracing.folded").unwrap();
-
-    tracing_subscriber::registry()
-        // .with(fmt_layer)
-        .with(flame_layer)
-        .init();
-    _guard
-}
-
 lazy_static::lazy_static! {
     pub static ref LOG_COUNTING_BY_MIN: Arc<RwLock<HashMap<String, LogCounter>>> = Arc::new(RwLock::new(HashMap::new()));
 }

backend/windmill-common/src/users.rs

@@ -10,6 +10,8 @@ pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
 pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev";
 pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev";
 
+pub const COOKIE_NAME: &str = "token";
+
 pub fn username_to_permissioned_as(user: &str) -> String {
     if user.contains('@') {
         user.to_string()

backend/windmill-common/src/worker.rs

@@ -97,6 +97,7 @@ lazy_static::lazy_static! {
     pub static ref MIN_VERSION: Arc<RwLock<Version>> = Arc::new(RwLock::new(Version::new(0, 0, 0)));
     pub static ref MIN_VERSION_IS_AT_LEAST_1_427: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
     pub static ref MIN_VERSION_IS_AT_LEAST_1_432: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
+    pub static ref MIN_VERSION_IS_AT_LEAST_1_440: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
 
     // Features flags:
     pub static ref DISABLE_FLOW_SCRIPT: bool = std::env::var("DISABLE_FLOW_SCRIPT").ok().is_some_and(|x| x == "1" || x == "true");
@@ -610,6 +611,7 @@ pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postg
 
     *MIN_VERSION_IS_AT_LEAST_1_427.write().await = min_version >= Version::new(1, 427, 0);
     *MIN_VERSION_IS_AT_LEAST_1_432.write().await = min_version >= Version::new(1, 432, 0);
+    *MIN_VERSION_IS_AT_LEAST_1_440.write().await = min_version >= Version::new(1, 440, 0);
 
     *MIN_VERSION.write().await = min_version.clone();
     min_version >= cur_version

backend/windmill-indexer/src/completed_runs_ee.rs

@@ -1,6 +1,6 @@
+use anyhow::anyhow;
 use sqlx::{Pool, Postgres};
 use windmill_common::error::Error;
-use anyhow::anyhow;
 
 #[derive(Clone)]
 pub struct IndexReader;
@@ -8,7 +8,7 @@ pub struct IndexReader;
 #[derive(Clone)]
 pub struct IndexWriter;
 
-pub async fn init_index(_db: Pool<Postgres>) -> Result<(IndexReader, IndexWriter), Error> {
+pub async fn init_index(_db: &Pool<Postgres>) -> Result<(IndexReader, IndexWriter), Error> {
     Err(anyhow!("Cannot initialize index: not in EE").into())
 }
 
@@ -16,6 +16,7 @@ pub async fn run_indexer(
     _db: Pool<Postgres>,
     mut _index_writer: IndexWriter,
     mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
-) {
+) -> Result<(), Error> {
     tracing::error!("Cannot run indexer: not in EE");
+    Err(anyhow!("Cannot run indexer: not in EE").into())
 }

backend/windmill-indexer/src/lib.rs

@@ -1,3 +1,3 @@
 pub mod completed_runs_ee;
-pub mod service_logs_ee;
 pub mod indexer_ee;
+pub mod service_logs_ee;

backend/windmill-indexer/src/service_logs_ee.rs

@@ -1,6 +1,6 @@
+use anyhow::anyhow;
 use sqlx::{Pool, Postgres};
 use windmill_common::error::Error;
-use anyhow::anyhow;
 
 #[derive(Clone)]
 pub struct ServiceLogIndexReader;
@@ -8,7 +8,10 @@ pub struct ServiceLogIndexReader;
 #[derive(Clone)]
 pub struct ServiceLogIndexWriter;
 
-pub async fn init_index(_db: Pool<Postgres>) -> Result<(ServiceLogIndexReader, ServiceLogIndexWriter), Error> {
+pub async fn init_index(
+    _db: &Pool<Postgres>,
+    mut _killpill_rx: tokio::sync::broadcast::Sender<()>,
+) -> Result<(ServiceLogIndexReader, ServiceLogIndexWriter), Error> {
     Err(anyhow!("Cannot initialize index: not in EE").into())
 }
 
@@ -16,6 +19,7 @@ pub async fn run_indexer(
     _db: Pool<Postgres>,
     mut _index_writer: ServiceLogIndexWriter,
     mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
-) {
+) -> Result<(), Error> {
     tracing::error!("Cannot run indexer: not in EE");
+    Err(anyhow!("Cannot run indexer: not in EE").into())
 }

backend/windmill-queue/Cargo.toml

@@ -44,4 +44,3 @@ axum.workspace = true
 serde_urlencoded.workspace = true
 regex.workspace = true
 backon.workspace = true
-opentelemetry.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -20,8 +20,6 @@ use reqwest::Client;
 use serde::{ser::SerializeMap, Serialize};
 use serde_json::{json, value::RawValue};
 use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction};
-#[cfg(feature = "benchmark")]
-use std::time::Instant;
 use tokio::{sync::RwLock, time::sleep};
 use ulid::Ulid;
 use uuid::Uuid;
@@ -30,7 +28,7 @@ use windmill_audit::ActionKind;
 
 use windmill_common::{
     auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
-    cache,
+    cache::{self, FlowData},
     db::{Authed, UserDB},
     error::{self, to_anyhow, Error},
     flow_status::{
@@ -50,8 +48,8 @@ use windmill_common::{
     utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt},
     worker::{
         to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES,
-        DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, NO_LOGS,
-        WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
+        DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432,
+        MIN_VERSION_IS_AT_LEAST_1_440, NO_LOGS, WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
     },
     DB, METRICS_ENABLED,
 };
@@ -121,7 +119,7 @@ const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler";
 const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
 const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
 const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
-#[cfg(feature = "enterprise")]
+#[cfg(any(feature = "enterprise", feature = "cloud"))]
 const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev";
 
 #[derive(Clone, Debug)]
@@ -2124,10 +2122,7 @@ fn fullpath_with_workspace(
     job_kind: &JobKind,
 ) -> String {
     let path = script_path.map(String::as_str).unwrap_or("tmp/main");
-    let is_flow = matches!(
-        job_kind,
-        &JobKind::Flow | &JobKind::FlowPreview | &JobKind::SingleScriptFlow | &JobKind::FlowNode
-    );
+    let is_flow = job_kind.is_flow();
     format!(
         "{}/{}/{}",
         workspace_id,
@@ -2492,6 +2487,16 @@ pub enum PushIsolationLevel<'c> {
     Transaction(Transaction<'c, Postgres>),
 }
 
+impl<'c> PushIsolationLevel<'c> {
+    async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
+        match self {
+            PushIsolationLevel::Isolated(db, authed) => Ok((db.begin(&authed).await?).into()),
+            PushIsolationLevel::IsolatedRoot(db) => Ok(db.begin().await?),
+            PushIsolationLevel::Transaction(tx) => Ok(tx),
+        }
+    }
+}
+
 #[macro_export]
 macro_rules! fetch_scalar_isolated {
     ( $query:expr, $tx:expr) => {
@@ -2518,7 +2523,7 @@ macro_rules! fetch_scalar_isolated {
 
 use sqlx::types::JsonRawValue;
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct PushArgsOwned {
     pub extra: Option<HashMap<String, Box<RawValue>>>,
     pub args: HashMap<String, Box<RawValue>>,
@@ -2845,9 +2850,10 @@ pub async fn push<'c, 'd>(
             concurrency_time_window_s,
             cache_ttl,
             dedicated_worker,
+            path,
         } => (
             Some(id.0),
-            None,
+            Some(path),
             None,
             JobKind::FlowScript,
             None,
@@ -2861,14 +2867,23 @@ pub async fn push<'c, 'd>(
             None,
         ),
         JobPayload::FlowNode { id, path } => {
-            let value = cache::flow::fetch_flow(_db, id).await?;
-            let status = Some(FlowStatus::new(&value));
+            let data = cache::flow::fetch_flow(_db, id).await?;
+            let value = data.value();
+            let status = Some(FlowStatus::new(value));
+            // Keep inserting `value` if not all workers are updated.
+            // Starting at `v1.440`, the value is fetched on pull from the flow node id.
+            let value_o = if !*MIN_VERSION_IS_AT_LEAST_1_440.read().await {
+                Some(value.clone())
+            } else {
+                // `raw_flow` is fetched on pull.
+                None
+            };
             (
                 Some(id.0),
                 Some(path),
                 None,
                 JobKind::FlowNode,
-                Some(value),
+                value_o,
                 status,
                 None,
                 None,
@@ -2987,7 +3002,7 @@ pub async fn push<'c, 'd>(
             Some(path),
             None,
             JobKind::FlowDependencies,
-            Some(flow_value.clone()),
+            Some(flow_value),
             None,
             None,
             None,
@@ -2998,22 +3013,24 @@ pub async fn push<'c, 'd>(
             None,
         ),
         JobPayload::FlowDependencies { path, dedicated_worker, version } => {
-            let value_json = fetch_scalar_isolated!(
-                sqlx::query_scalar!("SELECT value as \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow_version WHERE id = $1 LIMIT 1", &version),
-                tx
-            )?
-            .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
-            let value = serde_json::from_str::<FlowValue>(value_json.get()).map_err(|err| {
-                Error::InternalErr(format!(
-                    "could not convert json to flow for {path}: {err:?}"
-                ))
-            })?;
+            // Keep inserting `value` if not all workers are updated.
+            // Starting at `v1.440`, the value is fetched on pull from the version id.
+            let value_o = if !*MIN_VERSION_IS_AT_LEAST_1_440.read().await {
+                let mut ntx = tx.into_tx().await?;
+                // The version has been inserted only within the transaction.
+                let data = cache::flow::fetch_version(&mut *ntx, version).await?;
+                tx = PushIsolationLevel::Transaction(ntx);
+                Some(data.value().clone())
+            } else {
+                // `raw_flow` is fetched on pull.
+                None
+            };
             (
                 Some(version),
                 Some(path),
                 None,
                 JobKind::FlowDependencies,
-                Some(value.clone()),
+                value_o,
                 None,
                 None,
                 None,
@@ -3044,11 +3061,10 @@ pub async fn push<'c, 'd>(
 
             let flow_status: FlowStatus = match restarted_from {
                 Some(restarted_from_val) => {
-                    let (_, _, step_n, truncated_modules, _, user_states, cleanup_module) =
+                    let (_, _, _, step_n, truncated_modules, user_states, cleanup_module) =
                         restarted_flows_resolution(
                             _db,
                             workspace_id,
-                            Some(value.clone()),
                             restarted_from_val.flow_job_id,
                             restarted_from_val.step_id.as_str(),
                             restarted_from_val.branch_or_iteration_n,
@@ -3083,20 +3099,25 @@ pub async fn push<'c, 'd>(
                     FlowStatus::new(&value)
                 } // this is a new flow being pushed, flow_status is set to flow_value
             };
+            let concurrency_key = value.concurrency_key.clone();
+            let concurrent_limit = value.concurrent_limit;
+            let concurrency_time_window_s = value.concurrency_time_window_s;
+            let cache_ttl = value.cache_ttl.map(|x| x as i32);
+            let priority = value.priority;
             (
                 None,
                 path,
                 None,
                 JobKind::FlowPreview,
-                Some(value.clone()),
+                Some(value),
                 Some(flow_status),
                 None,
-                value.concurrency_key.clone(),
-                value.concurrent_limit.clone(),
-                value.concurrency_time_window_s,
-                value.cache_ttl.map(|x| x as i32),
+                concurrency_key,
+                concurrent_limit,
+                concurrency_time_window_s,
+                cache_ttl,
                 None,
-                value.priority,
+                priority,
             )
         }
         JobPayload::SingleScriptFlow {
@@ -3118,15 +3139,13 @@ pub async fn push<'c, 'd>(
             let flow_value = FlowValue {
                 modules: vec![FlowModule {
                     id: "a".to_string(),
-                    value: windmill_common::worker::to_raw_value(
-                        &windmill_common::flows::FlowModuleValue::Script {
-                            input_transforms: input_transforms,
-                            path: path.clone(),
-                            hash: Some(hash),
-                            tag_override: tag_override,
-                            is_trigger: None,
-                        },
-                    ),
+                    value: to_raw_value(&FlowModuleValue::Script {
+                        input_transforms,
+                        path: path.clone(),
+                        hash: Some(hash),
+                        tag_override,
+                        is_trigger: None,
+                    }),
                     stop_after_if: None,
                     stop_after_all_iters_if: None,
                     summary: None,
@@ -3143,22 +3162,24 @@ pub async fn push<'c, 'd>(
                 }],
                 same_worker: false,
                 failure_module: None,
-                concurrency_time_window_s: concurrency_time_window_s,
-                concurrent_limit: concurrent_limit,
+                concurrency_time_window_s,
+                concurrent_limit,
                 skip_expr: None,
                 cache_ttl: cache_ttl.map(|val| val as u32),
                 early_return: None,
                 concurrency_key: custom_concurrency_key.clone(),
-                priority: priority,
+                priority,
                 preprocessor_module: None,
             };
+            // this is a new flow being pushed, flow_status is set to flow_value:
+            let flow_status: FlowStatus = FlowStatus::new(&flow_value);
             (
                 None,
                 Some(path),
                 None,
                 JobKind::Flow,
-                Some(flow_value.clone()),
-                Some(FlowStatus::new(&flow_value)), // this is a new flow being pushed, flow_status is set to flow_value
+                Some(flow_value),
+                Some(flow_status),
                 None,
                 custom_concurrency_key,
                 concurrent_limit,
@@ -3169,51 +3190,46 @@ pub async fn push<'c, 'd>(
             )
         }
         JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => {
-            // Do not use the lite version unless all workers are updated.
-            let value_json = if *DISABLE_FLOW_SCRIPT || (!*MIN_VERSION_IS_AT_LEAST_1_432.read().await  && !*CLOUD_HOSTED) {
-                fetch_scalar_isolated!(
-                    sqlx::query_scalar!(
-                        "SELECT flow_version.value as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow 
-                        LEFT JOIN flow_version
-                            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-                        WHERE flow.path = $1 AND flow.workspace_id = $2",
-                        &path, &workspace_id
-                    ),
-                    tx
-                )
-            } else {
-                fetch_scalar_isolated!(
-                    sqlx::query_scalar!(
-                        "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow 
-                        LEFT JOIN flow_version
-                            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
-                        LEFT JOIN flow_version_lite 
-                            ON flow_version_lite.id = flow_version.id
-                        WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
-                        &path, &workspace_id
-                    ),
-                    tx
-                )
-            }?
+            let mut ntx = tx.into_tx().await?;
+            // Fetch the latest version of the flow.
+            let version = sqlx::query_scalar!(
+                "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"
+                FROM flow WHERE path = $1 AND workspace_id = $2",
+                &path,
+                &workspace_id
+            )
+            .fetch_optional(&mut *ntx)
+            .await?
             .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
-            let mut value = serde_json::from_str::<FlowValue>(value_json.get()).map_err(|err| {
-                Error::InternalErr(format!(
-                    "could not convert json to flow for {path}: {err:?}"
-                ))
-            })?;
-            let priority = value.priority;
-            add_virtual_items_if_necessary(&mut value.modules);
-            if same_worker {
-                value.same_worker = true;
-            }
-            let cache_ttl = value.cache_ttl.map(|x| x as i32).clone();
-            let custom_concurrency_key = value.concurrency_key.clone();
-            let concurrency_time_window_s = value.concurrency_time_window_s.clone();
-            let concurrent_limit = value.concurrent_limit.clone();
 
+            // Do not use the lite version unless all workers are updated.
+            let data = if *DISABLE_FLOW_SCRIPT
+                || (!*MIN_VERSION_IS_AT_LEAST_1_432.read().await && !*CLOUD_HOSTED)
+            {
+                cache::flow::fetch_version(&mut *ntx, version).await
+            } else {
+                // Fallback to the original version if the lite version is not found.
+                // This also prevent a race condition where the flow is run just after deploy and
+                // the lite version is still being created.
+                match cache::flow::fetch_version_lite(&mut *ntx, version).await {
+                    Ok(data) => Ok(data),
+                    Err(_) => cache::flow::fetch_version(&mut *ntx, version).await,
+                }
+            }?;
+            tx = PushIsolationLevel::Transaction(ntx);
+
+            let value = data.value().clone();
+            let priority = value.priority;
+            let cache_ttl = value.cache_ttl.map(|x| x as i32);
+            let custom_concurrency_key = value.concurrency_key.clone();
+            let concurrency_time_window_s = value.concurrency_time_window_s;
+            let concurrent_limit = value.concurrent_limit;
+
+            // this is a new flow being pushed, status is set to `value`.
+            let mut status = FlowStatus::new(&value);
             let extra = args.extra.get_or_insert_with(HashMap::new);
             if !apply_preprocessor {
-                value.preprocessor_module = None;
+                status.preprocessor_module = None;
                 extra.remove("wm_trigger");
             } else {
                 extra.entry("wm_trigger".to_string()).or_insert_with(|| {
@@ -3222,14 +3238,30 @@ pub async fn push<'c, 'd>(
                     }))
                 });
             }
-            let status = Some(FlowStatus::new(&value));
+            // Keep inserting `value` if not all workers are updated.
+            // Starting at `v1.440`, the value is fetched on pull from the version id.
+            let value_o = if !*MIN_VERSION_IS_AT_LEAST_1_440.read().await {
+                let mut value = value;
+                add_virtual_items_if_necessary(&mut value.modules);
+                if same_worker {
+                    value.same_worker = true;
+                }
+                if !apply_preprocessor {
+                    value.preprocessor_module = None;
+                }
+                Some(value)
+            } else {
+                // `raw_flow` is fetched on pull, the mutations from the other branch are replaced
+                // by additional checks when handling the flow.
+                None
+            };
             (
-                None,
+                Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull.
                 Some(path),
                 None,
                 JobKind::Flow,
-                Some(value),
-                status, // this is a new flow being pushed, flow_status is set to flow_value
+                value_o,
+                Some(status),
                 None,
                 custom_concurrency_key,
                 concurrent_limit,
@@ -3241,17 +3273,16 @@ pub async fn push<'c, 'd>(
         }
         JobPayload::RestartedFlow { completed_job_id, step_id, branch_or_iteration_n } => {
             let (
+                version,
                 flow_path,
-                raw_flow,
+                flow_data,
                 step_n,
                 truncated_modules,
-                priority,
                 user_states,
                 cleanup_module,
             ) = restarted_flows_resolution(
                 _db,
                 workspace_id,
-                None,
                 completed_job_id,
                 step_id.as_str(),
                 branch_or_iteration_n,
@@ -3280,18 +3311,32 @@ pub async fn push<'c, 'd>(
                 user_states,
                 preprocessor_module: None,
             };
+            let value = flow_data.value();
+            let priority = value.priority;
+            let concurrency_key = value.concurrency_key.clone();
+            let concurrent_limit = value.concurrent_limit;
+            let concurrency_time_window_s = value.concurrency_time_window_s;
+            let cache_ttl = value.cache_ttl.map(|x| x as i32);
+            // Keep inserting `value` if not all workers are updated.
+            // Starting at `v1.440`, the value is fetched on pull from the version id.
+            let value_o = if version.is_none() || !*MIN_VERSION_IS_AT_LEAST_1_440.read().await {
+                Some(value.clone())
+            } else {
+                // `raw_flow` is fetched on pull.
+                None
+            };
             (
-                None,
+                version,
                 flow_path,
                 None,
                 JobKind::Flow,
-                Some(raw_flow.clone()),
+                value_o,
                 Some(restarted_flow_status),
                 None,
-                raw_flow.concurrency_key,
-                raw_flow.concurrent_limit,
-                raw_flow.concurrency_time_window_s,
-                raw_flow.cache_ttl.map(|x| x as i32),
+                concurrency_key,
+                concurrent_limit,
+                concurrency_time_window_s,
+                cache_ttl,
                 None,
                 priority,
             )
@@ -3415,12 +3460,7 @@ pub async fn push<'c, 'd>(
         let interpolated_tag = tag.map(|x| interpolate_args(x, &args, workspace_id));
 
         let default = || {
-            let ntag = if job_kind == JobKind::Flow
-                || job_kind == JobKind::FlowPreview
-                || job_kind == JobKind::SingleScriptFlow
-                || job_kind == JobKind::Identity
-                || job_kind == JobKind::FlowNode
-            {
+            let ntag = if job_kind.is_flow() || job_kind == JobKind::Identity {
                 "flow".to_string()
             } else if job_kind == JobKind::Dependencies
                 || job_kind == JobKind::FlowDependencies
@@ -3457,11 +3497,7 @@ pub async fn push<'c, 'd>(
         })
     };
 
-    let mut tx = match tx {
-        PushIsolationLevel::Isolated(user_db, authed) => (user_db.begin(&authed).await?).into(),
-        PushIsolationLevel::IsolatedRoot(db) => db.begin().await?,
-        PushIsolationLevel::Transaction(tx) => tx,
-    };
+    let mut tx = tx.into_tx().await?;
 
     let job_id: Uuid = if let Some(job_id) = job_id {
         let conflicting_id = sqlx::query_scalar!(
@@ -3708,66 +3744,48 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
 async fn restarted_flows_resolution(
     db: &Pool<Postgres>,
     workspace_id: &str,
-    flow_value_if_any: Option<FlowValue>,
     completed_flow_id: Uuid,
     restart_step_id: &str,
     branch_or_iteration_n: Option<usize>,
 ) -> Result<
     (
+        Option<i64>,
         Option<String>,
-        FlowValue,
+        Arc<FlowData>,
         i32,
         Vec<FlowStatusModule>,
-        Option<i16>,
         HashMap<String, serde_json::Value>,
         FlowCleanupModule,
     ),
     Error,
 > {
-    #[derive(sqlx::FromRow)]
-    struct CompletedJobWithRawFlow {
-        #[sqlx(flatten)]
-        completed_job: CompletedJob,
-        raw_flow: Option<Json<Box<RawValue>>>,
-    }
+    let row = sqlx::query!(
+        "SELECT
+            script_path, script_hash AS \"script_hash: ScriptHash\",
+            job_kind AS \"job_kind: JobKind\",
+            flow_status AS \"flow_status: Json<Box<RawValue>>\",
+            raw_flow AS \"raw_flow: Json<Box<RawValue>>\"
+        FROM completed_job WHERE id = $1 and workspace_id = $2",
+        completed_flow_id,
+        workspace_id,
+    )
+    .fetch_one(db) // TODO: should we try to use the passed-in `tx` here?
+    .await
+    .map_err(|err| {
+        Error::InternalErr(format!(
+            "completed job not found for UUID {} in workspace {}: {}",
+            completed_flow_id, workspace_id, err
+        ))
+    })?;
 
-    let CompletedJobWithRawFlow { completed_job, raw_flow } =
-        sqlx::query_as::<_, CompletedJobWithRawFlow>(
-            "SELECT *, null as labels FROM completed_job WHERE id = $1 and workspace_id = $2",
-        )
-        .bind(completed_flow_id)
-        .bind(workspace_id)
-        .fetch_one(db) // TODO: should we try to use the passed-in `tx` here?
-        .await
-        .map_err(|err| {
-            Error::InternalErr(format!(
-                "completed job not found for UUID {} in workspace {}: {}",
-                completed_flow_id, workspace_id, err
-            ))
-        })?;
-
-    let flow_value = if let Some(flow_value) = flow_value_if_any {
-        Some(flow_value)
-    } else if let Some(raw_flow) = raw_flow.as_ref() {
-        serde_json::from_str::<FlowValue>(raw_flow.get()).ok()
-    } else {
-        sqlx::query_scalar!(
-            "SELECT raw_flow AS \"raw_flow!: Json<Box<JsonRawValue>>\"
-            FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1",
-            &completed_flow_id,
-            workspace_id
-        )
-        .fetch_one(db)
-        .await
-        .ok()
-        .and_then(|raw_flow| serde_json::from_str::<FlowValue>(raw_flow.get()).ok())
-    }
-    .ok_or(Error::InternalErr(format!(
-        "Unable to parse raw definition for job {} in workspace {}",
-        completed_flow_id, workspace_id,
-    )))?;
-    let flow_status = completed_job
-        .parse_flow_status()
+    let flow_data = cache::job::fetch_flow(db, row.job_kind, row.script_hash)
+        .or_else(|_| cache::job::fetch_preview_flow(db, &completed_flow_id, row.raw_flow))
+        .await?;
+    let flow_value = flow_data.value();
+    let flow_status = row
+        .flow_status
+        .as_ref()
+        .and_then(|v| serde_json::from_str::<FlowStatus>(v.get()).ok())
         .ok_or(Error::InternalErr(format!(
             "Unable to parse flow status for job {} in workspace {}",
             completed_flow_id, workspace_id,
@@ -3829,7 +3847,7 @@ async fn restarted_flows_resolution(
                                 branch: branch_or_iteration_n - 1, // Doing minus one here as this variable reflects the latest finished job in the iteration
                                 len: branches.len(),
                             }),
-                            parallel: parallel,
+                            parallel,
                             while_loop: false,
                             progress: None,
                         });
@@ -3867,7 +3885,7 @@ async fn restarted_flows_resolution(
                             flow_jobs_success: new_flow_jobs_success,
                             branch_chosen: None,
                             branchall: None,
-                            parallel: parallel,
+                            parallel,
                             while_loop: false,
                             progress: None,
                         });
@@ -3904,13 +3922,13 @@ async fn restarted_flows_resolution(
         )));
     }
 
-    return Ok((
-        completed_job.script_path,
-        flow_value,
+    Ok((
+        row.script_hash.map(|x| x.0),
+        row.script_path,
+        flow_data,
         step_n,
         truncated_modules,
-        completed_job.priority,
         flow_status.user_states,
         flow_status.cleanup_module,
-    ));
+    ))
 }