merge

* feat: kafka better retry and errors

* update ee ref

.github/workflows/backend-test.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: oven-sh/setup-bun@v2
         with:
           bun-version: 1.1.43
-      - uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@v6.2.1
         with:
           version: "0.6.2"
       - uses: actions-rust-lang/setup-rust-toolchain@v1

.github/workflows/change-versions.yml

@@ -9,7 +9,14 @@ jobs:
     runs-on: ubicloud
     container: node:18
     steps:
+      - uses: actions/create-github-app-token@v2
+        id: app
+        with:
+          app-id: ${{ vars.INTERNAL_APP_ID }}
+          private-key: ${{ secrets.INTERNAL_APP_KEY }}
       - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app.outputs.token }}
       - run: git config --system --add safe.directory /__w/windmill/windmill
       - name: Change versions
         run: ./.github/change-versions.sh "$(cat version.txt)"
@@ -21,3 +28,8 @@ jobs:
           cd backend
           cargo generate-lockfile
       - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_user_name: windmill-internal-app[bot]
+          commit_user_email: windmill-internal-app[bot]@users.noreply.github.com
+        env:
+          GITHUB_TOKEN: ${{ steps.app.outputs.token }}

.github/workflows/check-org-membership.yml

@@ -0,0 +1,60 @@
+name: Check Organization Membership
+
+on:
+  workflow_call:
+    inputs:
+      commenter:
+        required: true
+        type: string
+        description: 'The username to check for organization membership'
+      organization:
+        required: false
+        type: string
+        default: 'windmill-labs'
+        description: 'The organization to check membership for'
+      trusted_bot:
+        required: false
+        type: string
+        default: 'windmill-internal-app[bot]'
+        description: 'The trusted bot username to allow'
+    secrets:
+      access_token:
+        required: true
+        description: 'The access token to use for org membership check'
+    outputs:
+      is_member:
+        description: 'Whether the user is an organization member or trusted bot'
+        value: ${{ jobs.check-membership.outputs.is_member }}
+
+jobs:
+  check-membership:
+    runs-on: ubicloud-standard-2
+    outputs:
+      is_member: ${{ steps.check-membership.outputs.is_member }}
+    steps:
+      - name: Check organization membership
+        id: check-membership
+        env:
+          ORG_ACCESS_TOKEN: ${{ secrets.access_token }}
+          COMMENTER: ${{ inputs.commenter }}
+          ORG: ${{ inputs.organization }}
+          TRUSTED_BOT: ${{ inputs.trusted_bot }}
+        run: |
+          # 1. Allow the trusted bot straight away
+          if [[ "$COMMENTER" == "$TRUSTED_BOT" ]]; then
+            echo "is_member=true" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # 2. Otherwise fall back to the org-membership check
+          STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+            -H "Authorization: token $ORG_ACCESS_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/orgs/$ORG/members/$COMMENTER")
+
+          if [ "$STATUS" -eq 204 ]; then
+            echo "is_member=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_member=false" >> $GITHUB_OUTPUT
+          fi

.github/workflows/claude.yml

@@ -11,45 +11,40 @@ on:
     types: [submitted]
 
 jobs:
-  check-membership:
+  determine-commenter:
     if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/ai') && !contains(github.event.comment.user.login, '[bot]')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/ai') && !contains(github.event.review.user.login, '[bot]')) ||
-      (github.event_name == 'issues' && contains(github.event.issue.body, '/ai') && !contains(github.event.issue.user.login, '[bot]'))
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/ai')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/ai')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '/ai')) ||
+      (github.event_name == 'issues' && contains(github.event.issue.body, '/ai'))
     runs-on: ubicloud-standard-2
     outputs:
-      is_member: ${{ steps.check-membership.outputs.is_member }}
+      commenter: ${{ steps.determine-commenter.outputs.commenter }}
     steps:
-      - name: Check organization membership
-        id: check-membership
-        env:
-          ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }}
+      - name: Determine commenter
+        id: determine-commenter
         run: |
-          ORG="windmill-labs"
-
-          if [[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]]; then
+          # Work out who wrote the comment / review
+          if [[ "${{ github.event_name }}" == "issue_comment" || \
+                "${{ github.event_name }}" == "pull_request_review_comment" ]]; then
             COMMENTER="${{ github.event.comment.user.login }}"
           elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then
             COMMENTER="${{ github.event.review.user.login }}"
           else
             COMMENTER="${{ github.event.issue.user.login }}"
           fi
+          echo "commenter=$COMMENTER" >> $GITHUB_OUTPUT
 
-          STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-            -H "Authorization: token $ORG_ACCESS_TOKEN" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "https://api.github.com/orgs/$ORG/members/$COMMENTER")
-
-          if [ "$STATUS" -eq 204 ]; then
-            echo "is_member=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_member=false" >> $GITHUB_OUTPUT
-          fi
+  check-membership:
+    needs: determine-commenter
+    uses: ./.github/workflows/check-org-membership.yml
+    with:
+      commenter: ${{ needs.determine-commenter.outputs.commenter }}
+    secrets:
+      access_token: ${{ secrets.ORG_ACCESS_TOKEN }}
 
   claude-code-action:
-    needs: check-membership
+    needs: [determine-commenter, check-membership]
     if: |
       needs.check-membership.outputs.is_member == 'true'
     runs-on: ubicloud-standard-8
@@ -64,21 +59,69 @@ jobs:
         with:
           fetch-depth: 1
 
+      - uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Run npm install and generate-backend-client
+        working-directory: ./frontend
+        run: |
+          # add a build directory for cargo check
+          mkdir -p build
+          npm install
+          npm run generate-backend-client
+
+      - name: install xmlsec1
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libxml2-dev libxmlsec1-dev
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-workspaces: backend
+          toolchain: 1.85.0
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend
+
+      - name: cargo check
+        working-directory: ./backend
+        timeout-minutes: 16
+        run: |
+          SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh)
+
       - name: Run Claude PR Action
         uses: anthropics/claude-code-action@beta
+        env:
+          SQLX_OFFLINE: true
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           timeout_minutes: "60"
-          allowed_tools: "mcp__github__create_pull_request,Bash(npm:*),Bash(cargo:*)"
-          custom_instructions: "TEST IMPORTANT INSTRUCTIONS:
-          - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main, with the title starting with [Claude PR].
-          - If you made changes to the frontend code, run npm install, then npm run generate-backend-client, then npm run check. You can ignore warnings that are reported by the check script, but fix the errors.
-          - If you made changes to the backend code, run cargo check. You can ignore warnings that are reported by the check script, but fix the errors.
-          - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue.
-          AVAILABLE TOOLS:
-          - mcp__github__create_pull_request: Create a PR from a branch to main
-          - Bash(npm run check): Run the check script. You should run this tool after making changes to the frontend code.
-          - Bash(npm install): Install dependencies. You need this to run npm run check.
-          - Bash(npm run generate-backend-client): Generate the backend client. You need this to run npm run check.
-          - Bash(cargo check): Run the cargo check script. You should run this tool after making changes to the backend code.
+          allowed_tools: "mcp__github__create_pull_request,Bash"
+          custom_instructions: |
+            ## IMPORTANT INSTRUCTIONS
+            - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main.
+
+            ## Code Quality Requirements
+
+            After making any code changes, you MUST run the appropriate validation commands:
+
+            **Frontend Changes:**
+            - Run: `npm run check` in the frontend directory
+            - Fix all warnings and errors before proceeding
+
+            **Backend Changes:**  
+            - Run: `cargo check --features $(./all_features_oss.sh)` in the backend directory
+            - Fix all warnings and errors before proceeding
+
+            **Pull Request Creation:**
+            - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue. 
+
+            ## Available Tools
+            - mcp__github__create_pull_request: Create PRs from branches
+            - Bash: Full access to run validation commands and git operations
           trigger_phrase: "/ai"

.github/workflows/create-docs.yml

@@ -4,38 +4,36 @@ on:
 
 jobs:
   check-membership:
-    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') && github.event.comment.user.type != 'Bot' }}
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') }}
+    uses: ./.github/workflows/check-org-membership.yml
+    with:
+      commenter: ${{ github.event.comment.user.login }}
+    secrets:
+      access_token: ${{ secrets.ORG_ACCESS_TOKEN }}
+
+  generate-token:
+    needs: check-membership
+    if: ${{ needs.check-membership.outputs.is_member == 'true' }}
     runs-on: ubicloud-standard-2
     outputs:
-      is_member: ${{ steps.check-membership.outputs.is_member }}
+      app_token: ${{ steps.app.outputs.token }}
     steps:
-      - name: Check organization membership
-        id: check-membership
-        env:
-          ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }}
-          COMMENTER: ${{ github.event.comment.user.login }}
-        run: |
-          ORG="windmill-labs"
-          STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-            -H "Authorization: token $ORG_ACCESS_TOKEN" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "https://api.github.com/orgs/$ORG/members/$COMMENTER")
-
-          if [ "$STATUS" -eq 204 ]; then
-            echo "is_member=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_member=false" >> $GITHUB_OUTPUT
-          fi
+      - name: Generate an installation token
+        id: app
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.INTERNAL_APP_ID }}
+          private-key: ${{ secrets.INTERNAL_APP_KEY }}
+          owner: windmill-labs
 
   trigger-docs:
-    needs: check-membership
-    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') && needs.check-membership.outputs.is_member == 'true' }}
+    needs: [generate-token, check-membership]
+    if: ${{ needs.check-membership.outputs.is_member == 'true' }}
     uses: windmill-labs/windmilldocs/.github/workflows/create-docs.yml@main
     with:
       pr_number: ${{ github.event.issue.number }}
       repo: ${{ github.event.repository.name }}
       comment_text: ${{ github.event.comment.body }}
     secrets:
-      DOCS_TOKEN: ${{ secrets.DOCS_TOKEN }}
+      DOCS_TOKEN: ${{ needs.generate-token.outputs.app_token }}
       GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}

.github/workflows/discord-notification.yml

@@ -24,7 +24,7 @@ jobs:
       DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }}
 
   merge_success_emoji:
-    if: github.event.pull_request.merged == true
+    if: github.event.action == 'closed'
     uses: ./.github/workflows/shareable-discord-notification.yml
     with:
       PR_STATUS: "merged"

.github/workflows/docker-image.yml

@@ -7,7 +7,7 @@ env:
 name: Build windmill:main
 on:
   push:
-    branches: [main]
+    branches: [main, rf/migrateFromDepot]
     tags: ["*"]
   pull_request:
     types: [opened, synchronize, reopened]
@@ -35,69 +35,113 @@ concurrency:
 
 permissions: write-all
 
+# build:
+
 jobs:
   build:
-    runs-on: ubicloud
-    if: (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
-      !github.event.inputs.ee)
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          fetch-depth: 0
+    uses: ./.github/workflows/multi-build-arch.yml
+    with:
+      registry: ghcr.io
+      name: windmill
+      image_name:
+        ${{ github.event_name != 'pull_request' && github.event_name !=
+        'workflow_dispatch' && github.repository ||
+        'windmill-labs/windmill-test' }}
+      fetch_depth: 0
+      push_image: true
+      build_runner_cpus: 16
+      metadata_flavor: |
+        latest=false
+      metadata_labels: |
+        org.opencontainers.image.licenses=AGPLv3
+      build_args: |
+        features=embedding,parquet,jemalloc,deno_core,license,http_trigger,zip,dind,php,mysql,websocket,oauth2,python,smtp,csharp,static_frontend,rust
+      checkout_ee: true
+    secrets: inherit # that makes secrets available to the called workflow
 
-      - name: Read EE repo commit hash
-        run: |
-          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+    # runs-on: ubicloud
+    # if:
+    #   (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
+    #   !github.event.inputs.ee)
+    # steps:
+    #   - uses: actions/checkout@v4
+    #     with:
+    #       fetch-depth: 0
 
-      - uses: actions/checkout@v4
-        with:
-          repository: windmill-labs/windmill-ee-private
-          path: ./windmill-ee-private
-          ref: ${{ env.ee_repo_ref }}
-          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
-          fetch-depth: 0
+    #   - name: Read EE repo commit hash
+    #     run: |
+    #       echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
 
-      # - name: Set up Docker Buildx
-      #   uses: docker/setup-buildx-action@v2
-      - uses: depot/setup-action@v1
+    #   - uses: actions/checkout@v4
+    #     with:
+    #       repository: windmill-labs/windmill-ee-private
+    #       path: ./windmill-ee-private
+    #       ref: ${{ env.ee_repo_ref }}
+    #       token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+    #       fetch-depth: 0
 
-      - name: Login to registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+    #   # - name: Set up Docker Buildx
+    #   #   uses: docker/setup-buildx-action@v2
+    #   - uses: depot/setup-action@v1
 
-      - name: Substitute EE code (EE logic is behind feature flag)
-        run: |
-          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+    #   - name: Login to registry
+    #     uses: docker/login-action@v3
+    #     with:
+    #       registry: ${{ env.REGISTRY }}
+    #       username: ${{ github.actor }}
+    #       password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Docker meta
-        id: meta-public
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            latest=false
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
+    #   - name: Substitute EE code (EE logic is behind feature flag)
+    #     run: |
+    #       ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
 
-      - name: Build and push publicly
-        uses: depot/build-push-action@v1
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp,private
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
-            ${{ steps.meta-public.outputs.tags }}
-          labels: |
-            ${{ steps.meta-public.outputs.labels }}
+    #   - name: Docker meta
+    #     id: meta-public
+    #     uses: docker/metadata-action@v5
+    #     with:
+    #       images: |
+    #         ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+    #       flavor: |
+    #         latest=false
+    #       tags: |
+    #         type=semver,pattern={{version}}
+    #         type=semver,pattern={{major}}.{{minor}}
+
+    #   - name: Build and push publicly
+    #     uses: depot/build-push-action@v1
+    #     with:
+    #       context: .
+    #       platforms: linux/amd64,linux/arm64
+    #       push: true
+    #       build-args: |
+    #         features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,websocket,python,smtp,csharp,static_frontend,rust
+    #       tags: |
+    #         ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
+    #         ${{ steps.meta-public.outputs.tags }}
+    #       labels: |
+    #         ${{ steps.meta-public.outputs.labels }}
+    #         org.opencontainers.image.licenses=AGPLv3
+
+  build_ee_ubi:
+    uses: ./.github/workflows/multi-build-arch.yml
+    with:
+      registry: ghcr.io
+      name: windmill-ee
+      image_name:
+        ${{ github.event_name != 'pull_request' && github.event_name !=
+        'workflow_dispatch' && github.repository ||
+        'windmill-labs/windmill-ee-test' }}
+      fetch_depth: 0
+      push_image: true
+      build_runner_cpus: 16
+      metadata_flavor: |
+        latest=false
+      metadata_labels: |
+        org.opencontainers.image.licenses=AGPLv3
+      build_args: |
+        features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,websocket,python,smtp,csharp,static_frontend,rust
+      checkout_ee: true
+    secrets: inherit # that makes secrets available to the called workflow
 
   build_ee:
     runs-on: ubicloud
@@ -451,7 +495,8 @@ jobs:
     needs: [build_ee_nsjail]
     runs-on: ubicloud-standard-2-arm
     if: (github.event_name != 'pull_request') && (github.event_name !=
-      'workflow_dispatch')
+      'workflow_dispatch') && (github.ref == 'refs/heads/main' ||
+      startsWith(github.ref, 'refs/tags/v'))
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/helmchart_on_release.yml

@@ -9,11 +9,19 @@ jobs:
     runs-on: ubicloud-standard-2
 
     steps:
+      - name: Generate an installation token
+        id: app
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.INTERNAL_APP_ID }}
+          private-key: ${{ secrets.INTERNAL_APP_KEY }}
+          owner: windmill-labs
+
       - name: Checkout on helm repository
         uses: actions/checkout@v3
         with:
           repository: windmill-labs/windmill-helm-charts
-          token: ${{ secrets.HELM_CHART_TOKEN }}
+          token: ${{ steps.app.outputs.token }}
 
       - name: Get version
         id: get_version
@@ -49,6 +57,23 @@ jobs:
           APP_VERSION=${APP_VERSION%/}
           sed -i "s/appVersion: .*/appVersion: $APP_VERSION/" ./charts/windmill/Chart.yaml
 
+      - name: Close existing bump-helm PRs
+        env:
+          GH_TOKEN: ${{ steps.app.outputs.token }}
+        run: |
+          # List open PR numbers whose title starts with the prefix
+          prs=$(gh pr list \
+                  --state open \
+                  --search '"helm: bump version to" in:title' \
+                  --json number \
+                  -q '.[].number')
+
+          for pr in $prs; do
+            echo "Closing outdated bump PR #$pr"
+            gh pr close "$pr" \
+              --comment "Closed automatically – superseded by a newer Helm-chart bump PR."
+          done
+
       - name: Commit and push
         run: |
           git add .
@@ -57,7 +82,7 @@ jobs:
 
       - name: Create PR
         env:
-          GH_TOKEN: ${{ secrets.HELM_CHART_TOKEN }}
+          GH_TOKEN: ${{ steps.app.outputs.token }}
         run: |
           gh pr create \
             --title "helm: bump version to ${{ env.VERSION }}" \

.github/workflows/multi-build-arch.yml

@@ -0,0 +1,177 @@
+name: Build multi-platform Docker image
+
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      image_name:
+        required: true
+        type: string
+      name:
+        required: true
+        type: string
+      metadata_flavor:
+        default: |
+          latest=auto
+        type: string
+      metadata_tags:
+        type: string
+        default: |
+          type=schedule
+          type=semver,pattern={{version}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{major}}
+          type=ref,event=branch
+          type=ref,event=pr
+          type=sha
+      metadata_labels:
+        type: string
+        default: |
+          org.opencontainers.image.licenses=Windmill-Enterprise-License
+      build_args:
+        type: string
+        default: ""
+      push_image:
+        default: false
+        type: boolean
+      checkout_ee:
+        default: false
+        type: boolean
+      fetch_depth:
+        default: 1
+        type: number
+      after_checkout_command:
+        default: ""
+        type: string
+      build_runner_cpus:
+        default: 4
+        type: number
+
+jobs:
+  docker:
+    name: Docker ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubicloud-standard-${{ inputs.build_runner_cpus }}
+            platform: linux/amd64
+          - runner: ubicloud-standard-${{ inputs.build_runner_cpus }}-arm
+            platform: linux/arm64
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ inputs.fetch_depth }}
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        if: ${{ inputs.checkout_ee }}
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: ${{ inputs.fetch_depth }}
+
+      - name: Run command after checkout
+        if: ${{ inputs.after_checkout_command }}
+        run: ${{ inputs.after_checkout_command }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.registry }}/${{ inputs.image_name }}
+          tags: ${{ inputs.metadata_tags }}
+          labels: ${{ inputs.metadata_labels }}
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: ${{ inputs.build_args }}
+          cache-from: type=gha,scope=ubicloud-${{ matrix.runner }}-${{ inputs.name }}
+          cache-to: type=gha,mode=max,scope=ubicloud-${{ matrix.runner }}-${{ inputs.name }}
+          outputs: type=image,name=${{ inputs.registry }}/${{ inputs.image_name
+            }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Print cache proxy logs
+        run: sudo cat /var/log/cacheproxy.log
+        if: always()
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: ${{ inputs.push_image }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.runner }}-${{ inputs.name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    if: ${{ inputs.push_image }}
+    runs-on: ubicloud
+    needs:
+      - docker
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-*
+          merge-multiple: true
+          path: /tmp/digests
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.registry }}/${{ inputs.image_name }}
+          tags: ${{ inputs.metadata_tags }}
+          labels: ${{ inputs.metadata_labels }}
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ inputs.registry }}/${{ inputs.image_name }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image_name }}:${{ steps.meta.outputs.version }}

.github/workflows/pr-ready-review.yml

@@ -0,0 +1,22 @@
+name: Auto Comment on PR Ready for Review
+
+on:
+  pull_request:
+    types: [opened, ready_for_review]
+
+jobs:
+  add-review-comment:
+    if: github.event.pull_request.draft == false
+    runs-on: ubicloud-standard-2
+    steps:
+      - name: Add review comment
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.PUBLIC_REPO_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.pull_request.number,
+              body: '/ai review this PR'
+            });

.gitignore

@@ -11,4 +11,5 @@ CaddyfileRemoteMalo
 .dev-docker-wrapper*
 backend/.minio-data
 .aider*
-!.aiderignore
+!.aiderignore
+rust-client/Cargo.toml

CHANGELOG.md

@@ -1,5 +1,102 @@
 # Changelog
 
+## [1.501.4](https://github.com/windmill-labs/windmill/compare/v1.501.3...v1.501.4) (2025-06-26)
+
+
+### Bug Fixes
+
+* add windows paths to uv install to find git/ssh ([#6063](https://github.com/windmill-labs/windmill/issues/6063)) ([835f1d2](https://github.com/windmill-labs/windmill/commit/835f1d2ec945145942deaa41cb3bd176ed276279))
+* optionally enable CSP headers ([#6033](https://github.com/windmill-labs/windmill/issues/6033)) ([d933648](https://github.com/windmill-labs/windmill/commit/d933648d3666b2ca9d813e04b9f19ddc3c7efda3))
+* schemaform reorder ([#6069](https://github.com/windmill-labs/windmill/issues/6069)) ([1a4b096](https://github.com/windmill-labs/windmill/commit/1a4b096f3ce40e238f1724aa4fd26649d70cb62a))
+
+## [1.501.3](https://github.com/windmill-labs/windmill/compare/v1.501.2...v1.501.3) (2025-06-25)
+
+
+### Bug Fixes
+
+* **backend:** return correct content-type for openapi spec ([#6045](https://github.com/windmill-labs/windmill/issues/6045)) ([44457c7](https://github.com/windmill-labs/windmill/commit/44457c72cf75c969de97c39bb23f57acad268e10))
+* **frontend:** load all flow jobs on page load ([#6029](https://github.com/windmill-labs/windmill/issues/6029)) ([dc5e764](https://github.com/windmill-labs/windmill/commit/dc5e764d9db9251dc356094d6ac47c45fdf72c74))
+* ignore type only imports when computing ts lockfiles ([900c8ed](https://github.com/windmill-labs/windmill/commit/900c8edd7b35802e23a1359029da8ddbfb783753))
+* improve ordering of forms for non complete ordering + array schema fix ([18ee03a](https://github.com/windmill-labs/windmill/commit/18ee03a32371885f5e608cb306b5ccbccc31dac5))
+* missing static_asset_config from api call ([#6058](https://github.com/windmill-labs/windmill/issues/6058)) ([395f1ff](https://github.com/windmill-labs/windmill/commit/395f1ff8ba05020d72d1d8b34bd6bb32517b7aec))
+
+## [1.501.2](https://github.com/windmill-labs/windmill/compare/v1.501.1...v1.501.2) (2025-06-24)
+
+
+### Bug Fixes
+
+* improve schema form handling of inconsistent order and properties ([3daf79f](https://github.com/windmill-labs/windmill/commit/3daf79ffbc45ca32ff443e5521a67d62528665db))
+
+## [1.501.1](https://github.com/windmill-labs/windmill/compare/v1.501.0...v1.501.1) (2025-06-24)
+
+
+### Bug Fixes
+
+* optimize jobs list run incremental refresh performance ([1bdd00a](https://github.com/windmill-labs/windmill/commit/1bdd00a3e4a94ecb23efb9614c341c64a67ac389))
+* pwsh skip already installed modules outside of cache ([#6037](https://github.com/windmill-labs/windmill/issues/6037)) ([29f6fab](https://github.com/windmill-labs/windmill/commit/29f6fab60c6f8cf251182a56c09bac7692868bae))
+
+## [1.501.0](https://github.com/windmill-labs/windmill/compare/v1.500.3...v1.501.0) (2025-06-24)
+
+
+### Features
+
+* ai flow chat prompt and UX improvements ([#5942](https://github.com/windmill-labs/windmill/issues/5942)) ([5722014](https://github.com/windmill-labs/windmill/commit/57220146513444436faff95f58c1b36481d1fa1d))
+
+
+### Bug Fixes
+
+* improve reactivity of apps ([27e12a1](https://github.com/windmill-labs/windmill/commit/27e12a1527c41ac801042038b707a94897e718f8))
+
+## [1.500.3](https://github.com/windmill-labs/windmill/compare/v1.500.2...v1.500.3) (2025-06-23)
+
+
+### Bug Fixes
+
+* fix conditional wrappre ([6f3cb5e](https://github.com/windmill-labs/windmill/commit/6f3cb5eabb7b2224d04ec10f151f67c0955a5cfd))
+
+## [1.500.2](https://github.com/windmill-labs/windmill/compare/v1.500.1...v1.500.2) (2025-06-20)
+
+
+### Bug Fixes
+
+* consistency of root job propagation fixing cases where runFlow in scripts would fail ([9c2f6a7](https://github.com/windmill-labs/windmill/commit/9c2f6a757fb168c7305c991c9fdbf78acd856a1c))
+
+## [1.500.1](https://github.com/windmill-labs/windmill/compare/v1.500.0...v1.500.1) (2025-06-20)
+
+
+### Bug Fixes
+
+* git repository resource picker effect loop ([#6017](https://github.com/windmill-labs/windmill/issues/6017)) ([1b1bee5](https://github.com/windmill-labs/windmill/commit/1b1bee5b53d78e4407b684b567d0fddd2b5283f3))
+
+## [1.500.0](https://github.com/windmill-labs/windmill/compare/v1.499.0...v1.500.0) (2025-06-20)
+
+
+### Features
+
+* add typescript client context to ai chat system prompt ([#6004](https://github.com/windmill-labs/windmill/issues/6004)) ([3e82282](https://github.com/windmill-labs/windmill/commit/3e822823519d1d5c22e422e4bd1ad4d37b6428b6))
+* blacklist remote agent worker token ([#5985](https://github.com/windmill-labs/windmill/issues/5985)) ([86eb907](https://github.com/windmill-labs/windmill/commit/86eb9074cc94f309f17ea72e9cecd0d502ffd2be))
+* **frontend:** run steps from graph ([#5915](https://github.com/windmill-labs/windmill/issues/5915)) ([67e6bce](https://github.com/windmill-labs/windmill/commit/67e6bce9b2eba1653450921afab3eabbd41fc715))
+
+
+### Bug Fixes
+
+* ai button in inline script editor to open AI chat in flow builder ([#5989](https://github.com/windmill-labs/windmill/issues/5989)) ([4ae5928](https://github.com/windmill-labs/windmill/commit/4ae5928788831196672e212b32ca410afab640e0))
+* improve piptar upload - sequential uploads via background task queue ([#5994](https://github.com/windmill-labs/windmill/issues/5994)) ([c4adaee](https://github.com/windmill-labs/windmill/commit/c4adaeeabd287ca1c4f3522bcd8bcea30b00fe6d))
+* new MultiSelect component ([#5979](https://github.com/windmill-labs/windmill/issues/5979)) ([fa8d1b4](https://github.com/windmill-labs/windmill/commit/fa8d1b47db19e15fe854e01f9987c8f97cb45b44))
+* replace worker tags to listen multiselect ([#5997](https://github.com/windmill-labs/windmill/issues/5997)) ([e4255e6](https://github.com/windmill-labs/windmill/commit/e4255e6276565c4a45b1f45a5d627bcfb5369270))
+
+## [1.499.0](https://github.com/windmill-labs/windmill/compare/v1.498.0...v1.499.0) (2025-06-18)
+
+
+### Features
+
+* devOps role can edit worker groups ([#5984](https://github.com/windmill-labs/windmill/issues/5984)) ([b1c4f8b](https://github.com/windmill-labs/windmill/commit/b1c4f8b29d0fb4cad76853110b84a87892b54661))
+
+
+### Bug Fixes
+
+* prevent keypress events from bubbling in decision tree drawer ([#5993](https://github.com/windmill-labs/windmill/issues/5993)) ([2a33442](https://github.com/windmill-labs/windmill/commit/2a334421e85abf046784aab57522582439ef2901))
+
 ## [1.498.0](https://github.com/windmill-labs/windmill/compare/v1.497.2...v1.498.0) (2025-06-17)
 
 

CLAUDE.md

@@ -1,3 +1,10 @@
-To have an overview of what this app does, see @.cursor/rules/windmill-overview.mdc
-For backend modifications, follow the rules mentioned here @.cursor/rules/rust-best-practices.mdc. You also have access to a summarized version of the database schema here @backend/summarized_schema.txt
-For frontend modifications, follow the rules mentioned here @.cursor/rules/svelte5-best-practices.mdc
+# Windmill Development Guide
+
+## Overview
+
+Windmill is an open-source developer platform for building internal tools, workflows, API integrations, background jobs, workflows, and user interfaces. See @windmill-overview.mdc for full platform details.
+
+## Language-Specific Guides
+
+- Backend (Rust): @backend/rust-best-practices.mdc + @backend/summarized_schema.txt
+- Frontend (Svelte 5): @frontend/svelte5-best-practices.mdc

Dockerfile

@@ -1,5 +1,5 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
-ARG RUST_IMAGE=rust:1.86-slim-bookworm
+ARG RUST_IMAGE=rust:1.88-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
 
@@ -80,7 +80,7 @@ COPY .git/ .git/
 
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
-    CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
+    CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --jobs $(nproc) --release --features "$features"
 
 
 FROM ${DEBIAN_IMAGE}

backend/.sqlx/query-019100d178129340a7c35d60ab61f983c8a9cb810db4369554bf26c6b0d6003d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n                    id As \"id!\",\n                    flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json<Uuid>\"\n                FROM v2_as_queue\n                WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id AND workspace_id = $2",
+  "query": "SELECT\n                    id As \"id!\",\n                    flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json<Uuid>\"\n                FROM v2_job_status\n                WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id",
   "describe": {
     "columns": [
       {
@@ -16,14 +16,13 @@
     ],
     "parameters": {
       "Left": [
-        "Uuid",
-        "Text"
+        "Uuid"
       ]
     },
     "nullable": [
-      true,
+      false,
       null
     ]
   },
-  "hash": "3c0b2a840102b12864c5d721b8e0142602ab37f3e1a95d39b3c7cbd7ff34d0b2"
+  "hash": "019100d178129340a7c35d60ab61f983c8a9cb810db4369554bf26c6b0d6003d"
 }

backend/.sqlx/query-08c827d9b2de0b77ce0ea2653760751615112c501b35e931ed817dbefd7c6bdb.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM app WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "08c827d9b2de0b77ce0ea2653760751615112c501b35e931ed817dbefd7c6bdb"
+}

backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n             FROM agent_token_blacklist \n             ORDER BY blacklisted_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "expires_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 2,
+        "name": "blacklisted_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 3,
+        "name": "blacklisted_by",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "1c5d3556fc8436ddd294f39c5431e1f501a821d6143c5d8aece20814237a6b86"
+}

backend/.sqlx/query-222e29b89d10f3840d4e9b9ab63207df3cbab63c83d4a6374e72a11893841653.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "CREATE INDEX CONCURRENTLY idx_audit_recent_login_activities \nON audit (timestamp, username) \nWHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "222e29b89d10f3840d4e9b9ab63207df3cbab63c83d4a6374e72a11893841653"
+}

backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM agent_token_blacklist WHERE token = $1 AND expires_at > $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Timestamp"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "2bf99d540365c228e1776ee5d2ba01ebe289183526afab19c1390bbf5082f019"
+}

backend/.sqlx/query-2f30274b0fe89aa1579b252b990876e5035ca5b31a68fcf08701102a6457e5c4.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM token WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "2f30274b0fe89aa1579b252b990876e5035ca5b31a68fcf08701102a6457e5c4"
+}

backend/.sqlx/query-3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562"
+}

backend/.sqlx/query-52032730f2eeaaeab55305f72bea5481d1c50c2eaa92a97a078239430f0d6c13.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM flow WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "52032730f2eeaaeab55305f72bea5481d1c50c2eaa92a97a078239430f0d6c13"
+}

backend/.sqlx/query-54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM agent_token_blacklist WHERE token = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "54fee31b61d62598c89cf7d0729079ac1721fe7bd1844f339236379211defc78"
+}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-5a31b32659a0ac6a6ad0e122a4d475787240d6714ddadf16296d2b7bd5fdcb52.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM variable WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "5a31b32659a0ac6a6ad0e122a4d475787240d6714ddadf16296d2b7bd5fdcb52"
+}

backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-7c765f50c67b0ef751bafc1bf9279c4cb8a851dfab406ba7611f77773663e9f3.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM resource WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "7c765f50c67b0ef751bafc1bf9279c4cb8a851dfab406ba7611f77773663e9f3"
+}

backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "995b194da28092d5aa053df936e7a9ee4b80cf3ade038a032c57ecff8fa3c6cf"
+}

backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-9d488c5ba4b9f5203692721d76ec831f5954861a5576e0d8c1c42a9eca90927f.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM workspace WHERE owner = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9d488c5ba4b9f5203692721d76ec831f5954861a5576e0d8c1c42a9eca90927f"
+}

backend/.sqlx/query-b46a0fbebdc8e5e9852a06444b0aeaa4eaf67959e68b69eb2f0896ebe9244691.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT leaf_jobs->$1::text AS \"leaf_jobs: Json<Box<RawValue>>\", parent_job\n        FROM v2_as_queue\n        WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = id AND workspace_id = $3",
+  "query": "SELECT flow_leaf_jobs->$1::text AS \"leaf_jobs: Json<Box<RawValue>>\", v2_job.parent_job\n        FROM v2_job_status\n        LEFT JOIN v2_job ON v2_job.id = v2_job_status.id AND v2_job.workspace_id = $3\n        WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = v2_job_status.id",
   "describe": {
     "columns": [
       {
@@ -26,5 +26,5 @@
       true
     ]
   },
-  "hash": "cf12a70e7b75ae471a0944de34502384be156cf25129f9c52bda34b240cf469a"
+  "hash": "b46a0fbebdc8e5e9852a06444b0aeaa4eaf67959e68b69eb2f0896ebe9244691"
 }

backend/.sqlx/query-b7c3a66c3831eb5d145ff00807badae57bef81be051f150df754fd1444d7356d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH inserted_job AS (\n            INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n                created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n            script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n            flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n            cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $21, $22, $23, $24, $25, $26,\n            CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n        ),\n        inserted_runtime AS (\n            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n        ),\n        inserted_job_perms AS (\n            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n            values ($1, $32, $33, $34, $35, $36, $37, $2) \n            ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2\n        )\n        INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 THEN now() END, $30, $31)",
+  "query": "WITH inserted_job AS (\n            INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n                created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n            script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n            flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n            cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $38, $21, $22, $23, $24, $25, $26,\n            CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n        ),\n        inserted_runtime AS (\n            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n        ),\n        inserted_job_perms AS (\n            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n            values ($1, $32, $33, $34, $35, $36, $37, $2) \n            ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2\n        )\n        INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 THEN now() END, $30, $31)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -97,10 +97,11 @@
         "Bool",
         "Bool",
         "JsonbArray",
-        "TextArray"
+        "TextArray",
+        "Uuid"
       ]
     },
     "nullable": []
   },
-  "hash": "cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6"
+  "hash": "b7c3a66c3831eb5d145ff00807badae57bef81be051f150df754fd1444d7356d"
 }

backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO agent_token_blacklist (token, expires_at, blacklisted_by) \n         VALUES ($1, $2, $3) \n         ON CONFLICT (token) DO UPDATE SET \n            expires_at = EXCLUDED.expires_at,\n            blacklisted_at = NOW(),\n            blacklisted_by = EXCLUDED.blacklisted_by",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Timestamp",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c9c040ec228a8fe4fda08439420141bee63339d4e3d5e2d68aabb12009f691c6"
+}

backend/.sqlx/query-cb8bde4d92a020278cbae79c5c01a766c198392aceb38fb27e57b73de8f7f279.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM script WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "cb8bde4d92a020278cbae79c5c01a766c198392aceb38fb27e57b73de8f7f279"
+}

backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT token, expires_at, blacklisted_at, blacklisted_by \n             FROM agent_token_blacklist \n             WHERE expires_at > $1 \n             ORDER BY blacklisted_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "expires_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 2,
+        "name": "blacklisted_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 3,
+        "name": "blacklisted_by",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamp"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "d56722c25877222af9affd5da5bb83b28fa8cbb528a2cfc90684cb10a69e4375"
+}

backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/CLAUDE.md

@@ -0,0 +1,12 @@
+# Backend Development (Rust)
+
+## Core Principles
+
+- Follow @rust-best-practices.mdc for detailed guidelines
+- Database schema reference: @summarized_schema.txt
+- The API routes prefixes are all listed in windmill-api/src/lib.rs
+
+## Adding New Features
+
+1. Update database schema with migration if necessary
+2. Update backend/windmill-api/openapi.yaml after modifying API endpoints

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.498.0"
+version = "1.501.4"
 authors.workspace = true
 edition.workspace = true
 
@@ -32,7 +32,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.498.0"
+version = "1.501.4"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -329,7 +329,7 @@ jsonwebtoken = "8.3.0"
 pem = "3.0.1"
 nix = { version = "0.27.1", features = ["process", "signal"] }
 tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" }
-hf-hub = "0.3.2"
+hf-hub = "0.4.3"
 tokenizers = "0.14.1"
 candle-core = "0.9.1"
 candle-transformers = "0.9.1"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-67e727c618cf673850a0887931c803241abfcfe8
+651b945d4567081968005278d7e87ea41cabbe1c

backend/migrations/20250618164000_add_agent_token_blacklist.down.sql

@@ -0,0 +1,2 @@
+-- Remove agent token blacklist table
+DROP TABLE IF EXISTS agent_token_blacklist;

backend/migrations/20250618164000_add_agent_token_blacklist.up.sql

@@ -0,0 +1,14 @@
+-- Add agent token blacklist table
+CREATE TABLE agent_token_blacklist (
+    token VARCHAR PRIMARY KEY,
+    expires_at TIMESTAMP NOT NULL,
+    blacklisted_at TIMESTAMP NOT NULL DEFAULT NOW(),
+    blacklisted_by VARCHAR NOT NULL
+);
+
+-- Add index for efficient expiry cleanup
+CREATE INDEX idx_agent_token_blacklist_expires_at ON agent_token_blacklist(expires_at);
+
+-- Grant permissions to windmill users
+GRANT ALL ON agent_token_blacklist TO windmill_user;
+GRANT ALL ON agent_token_blacklist TO windmill_admin;

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -30,8 +30,6 @@ use windmill_common::{
     worker::PythonAnnotations,
 };
 
-const DEF_MAIN: &str = "def main(";
-
 fn replace_import(x: String) -> String {
     SHORT_IMPORTS_MAP
         .get(&x)
@@ -48,6 +46,9 @@ lazy_static! {
     static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap();
     static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap();
     static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap();
+    // Regex to properly match main function definition at line start,
+    // capturing both sync and async variants
+    static ref DEF_MAIN_RE: Regex = Regex::new(r"(?m)^(async\s+)?def\s+main\s*\(").unwrap();
 }
 
 fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<NImport> {
@@ -143,7 +144,12 @@ struct ImportPin {
 }
 
 fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<NImport>> {
-    let mut code = code.split(DEF_MAIN).next().unwrap_or("").to_string();
+    // Use regex to safely find the main function definition
+    let mut code = DEF_MAIN_RE
+        .split(code)
+        .next()
+        .unwrap_or_default()
+        .to_string();
 
     // remove main function decorator from end of file if it exists
     if code
@@ -160,10 +166,17 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<NImport>>
             + "\n";
     }
 
-    let ast = Suite::parse(&code, "main.py").map_err(|e| {
+    // Add a fake main function to ensure the parser can process the code correctly
+    // This is needed because we've split off the real main function above
+    let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
+
+    let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
         error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string()))
     })?;
 
+    // Note: We're still using the original code for finding pins,
+    // as the TextRange values from the parsed AST would be based on code_with_fake_main
+    // but we want to match against the original code
     let find_pin = |range: TextRange, key: String| {
         let hs = code
             .chars()
@@ -378,7 +391,7 @@ async fn parse_python_imports_inner(
                 })
                 .join("\n")
                 .parse::<toml::Table>()
-            .map_err(to_anyhow)?;
+                .map_err(to_anyhow)?;
 
             {
                 if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) {

backend/parsers/windmill-parser-py/src/lib.rs

@@ -232,7 +232,14 @@ fn parse_typ(id: &str) -> Typ {
         x @ _ if x.starts_with("DynSelect_") => {
             Typ::DynSelect(x.strip_prefix("DynSelect_").unwrap().to_string())
         }
-        _ => Typ::Resource(id.to_string()),
+        _ => Typ::Resource(map_resource_name(id)),
+    }
+}
+
+fn map_resource_name(x: &str) -> String {
+    match x {
+        "S3Object" => "s3_object".to_string(),
+        _ => x.to_string(),
     }
 }
 
@@ -468,7 +475,7 @@ def main(test1: str,
                     Arg {
                         otyp: None,
                         name: "s3o".to_string(),
-                        typ: Typ::Resource("S3Object".to_string()),
+                        typ: Typ::Resource("s3_object".to_string()),
                         default: None,
                         has_default: false,
                         oidx: None

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -29,12 +29,36 @@ use wasm_bindgen::prelude::*;
 
 struct ImportsFinder {
     imports: HashSet<String>,
+    skip_type_only: bool,
 }
 
 impl Visit for ImportsFinder {
     noop_visit_type!();
 
     fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) {
+        if self.skip_type_only {
+            if n.type_only {
+                return;
+            }
+            if n.specifiers.len() > 0 {
+                let mut is_type_only = true;
+
+                for specifier in n.specifiers.iter() {
+                    match specifier {
+                        swc_ecma_ast::ImportSpecifier::Named(
+                            swc_ecma_ast::ImportNamedSpecifier { is_type_only, .. },
+                        ) if *is_type_only => (),
+                        _ => {
+                            is_type_only = false;
+                            break;
+                        }
+                    }
+                }
+                if is_type_only {
+                    return;
+                }
+            }
+        }
         if let Some(ref s) = n.src.raw {
             let s = s.to_string();
             if s.starts_with("'") && s.ends_with("'") {
@@ -46,7 +70,7 @@ impl Visit for ImportsFinder {
     }
 }
 
-pub fn parse_expr_for_imports(code: &str) -> anyhow::Result<Vec<String>> {
+pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result<Vec<String>> {
     let cm: Lrc<SourceMap> = Default::default();
     let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into());
     let mut tss = TsSyntax::default();
@@ -72,7 +96,7 @@ pub fn parse_expr_for_imports(code: &str) -> anyhow::Result<Vec<String>> {
         anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}")
     })?;
 
-    let mut visitor = ImportsFinder { imports: HashSet::new() };
+    let mut visitor = ImportsFinder { imports: HashSet::new(), skip_type_only };
     visitor.visit_module(&expr);
 
     let mut imports: Vec<_> = visitor.imports.into_iter().collect();
@@ -318,7 +342,7 @@ lazy_static::lazy_static! {
 }
 
 pub fn remove_pinned_imports(code: &str) -> anyhow::Result<String> {
-    let mut imports = parse_expr_for_imports(code)?;
+    let mut imports = parse_expr_for_imports(code, false)?;
     imports.sort_by_key(|f| 0 - (f.len() as i32));
     let mut content = code.to_string();
     for import in imports {

backend/parsers/windmill-parser-ts/tests/tests.rs

@@ -0,0 +1,19 @@
+mod tests {
+    use windmill_parser_ts::parse_expr_for_imports;
+
+    #[test]
+    fn test_imports() {
+        let code = r#"
+        import { foo } from "bar";
+        import type { foo } from "bar2";
+        import { type foo, bar } from "bar3";
+                import { bar, type foo } from "bar7";
+
+        import { type foo, type bar } from "bar4";
+        import * as foo from "bar5";
+        import foo from "bar6";
+        "#;
+        let imports = parse_expr_for_imports(code, true).unwrap();
+        assert_eq!(imports, vec!["bar", "bar3", "bar5", "bar6", "bar7"]);
+    }
+}

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -41,7 +41,7 @@ pub fn parse_outputs(code: &str) -> String {
 #[cfg(feature = "ts-parser")]
 #[wasm_bindgen]
 pub fn parse_ts_imports(code: &str) -> String {
-    let parsed = parse_expr_for_imports(code);
+    let parsed = parse_expr_for_imports(code, false);
     let r = if let Ok(parsed) = parsed {
         json!({ "imports": parsed })
     } else {

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -339,7 +339,7 @@ fn test_parse_imports() -> anyhow::Result<()> {
     import { bar } from \"bar/foo/d\";
     import { bar as baroof } from \"bar\";
 ";
-    let mut l = parse_expr_for_imports(code)?;
+    let mut l = parse_expr_for_imports(code, false)?;
     l.sort();
     assert_eq!(
         l,
@@ -360,7 +360,7 @@ fn test_parse_imports_dts() -> anyhow::Result<()> {
     let code = "
 export type foo = number
 ";
-    let mut l = parse_expr_for_imports(code)?;
+    let mut l = parse_expr_for_imports(code, false)?;
     l.sort();
     assert_eq!(l, vec![] as Vec<String>);
 

backend/parsers/windmill-parser/src/lib.rs

@@ -92,6 +92,12 @@ mod test {
     fn test_snake_case() {
         assert_eq!("s3", to_snake_case("S3"));
         assert_eq!("s3", to_snake_case("s3"));
+        assert_eq!("s3_object", to_snake_case("S3Object"));
+        assert_eq!("s3_object", to_snake_case("S3object"));
+        assert_eq!("s3_object", to_snake_case("s3object"));
+        assert_eq!("abc", to_snake_case("ABC"));
+        assert_eq!("aa_bc", to_snake_case("AaBC"));
+        assert_eq!("a_b_c", to_snake_case("A_B_C"));
         assert_eq!("s_3", to_snake_case("S_3"));
         assert_eq!("type_name_here", to_snake_case("typeNameHere"));
     }

backend/src/main.rs

@@ -1102,6 +1102,9 @@ Windmill Community Edition {GIT_VERSION}
                         _ = tokio::time::sleep(Duration::from_secs(12 * 60 * 60)) => {
                             tracing::info!("Reloading config after 12 hours");
                             initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await;
+                            if let Err(e) = reload_license_key(&conn).await {
+                                tracing::error!("Failed to reload license key on agent: {e:#}");
+                            }
                             #[cfg(feature = "enterprise")]
                             ee_oss::verify_license_key().await;
                         }

backend/src/monitor.rs

@@ -840,6 +840,24 @@ pub async fn delete_expired_items(db: &DB) -> () {
         tracing::error!("Error deleting audit log on CE: {:?}", e);
     }
 
+    match sqlx::query_scalar!(
+        "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token",
+    )
+    .fetch_all(db)
+    .await
+    {
+        Ok(deleted_tokens) => {
+            if deleted_tokens.len() > 0 {
+                tracing::info!(
+                    "deleted {} expired blacklisted agent tokens: {:?}",
+                    deleted_tokens.len(),
+                    deleted_tokens
+                );
+            }
+        }
+        Err(e) => tracing::error!("Error deleting expired blacklisted agent tokens: {:?}", e),
+    }
+
     let job_retention_secs = *JOB_RETENTION_SECS.read().await;
     if job_retention_secs > 0 {
         match db.begin().await {

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.498.0
+  version: 1.501.4
   title: Windmill API
 
   contact:
@@ -11240,6 +11240,98 @@ paths:
               schema:
                 type: string
 
+  /agent_workers/blacklist_token:
+    post:
+      summary: blacklist agent token (requires super admin)
+      operationId: blacklistAgentToken
+      tags:
+        - agent_workers
+      requestBody:
+        description: token to blacklist
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                token:
+                  type: string
+                  description: The agent token to blacklist
+                expires_at:
+                  type: string
+                  format: date-time
+                  description: Optional expiration date for the blacklist entry
+              required:
+                - token
+      responses:
+        "200":
+          description: token blacklisted successfully
+
+  /agent_workers/remove_blacklist_token:
+    post:
+      summary: remove agent token from blacklist (requires super admin)
+      operationId: removeBlacklistAgentToken
+      tags:
+        - agent_workers
+      requestBody:
+        description: token to remove from blacklist
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                token:
+                  type: string
+                  description: The agent token to remove from blacklist
+              required:
+                - token
+      responses:
+        "200":
+          description: token removed from blacklist successfully
+
+  /agent_workers/list_blacklisted_tokens:
+    get:
+      summary: list blacklisted agent tokens (requires super admin)
+      operationId: listBlacklistedAgentTokens
+      tags:
+        - agent_workers
+      parameters:
+        - name: include_expired
+          in: query
+          description: Whether to include expired blacklisted tokens
+          schema:
+            type: boolean
+            default: false
+      responses:
+        "200":
+          description: list of blacklisted tokens
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    token:
+                      type: string
+                      description: The blacklisted token (without prefix)
+                    expires_at:
+                      type: string
+                      format: date-time
+                      description: When the blacklist entry expires
+                    blacklisted_at:
+                      type: string
+                      format: date-time
+                      description: When the token was blacklisted
+                    blacklisted_by:
+                      type: string
+                      description: Email of the user who blacklisted the token
+                  required:
+                    - token
+                    - expires_at
+                    - blacklisted_at
+                    - blacklisted_by
 
   /w/{workspace}/acls/get/{kind}/{path}:
     get:

backend/windmill-api/src/agent_workers_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-api/src/agent_workers_ee.rs

backend/windmill-api/src/agent_workers_oss.rs

@@ -16,9 +16,6 @@ use crate::db::DB;
 #[cfg(not(feature = "private"))]
 use axum::Router;
 
-#[cfg(not(feature = "private"))]
-use serde::{Deserialize, Serialize};
-
 #[cfg(not(feature = "private"))]
 pub fn global_service() -> Router {
     Router::new()
@@ -44,15 +41,6 @@ pub fn workspaced_service(
     (router, vec![], Some(job_completed_tx))
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[cfg(not(feature = "private"))]
-pub struct AgentAuth {
-    pub worker_group: String,
-    pub suffix: Option<String>,
-    pub tags: Vec<String>,
-    pub exp: Option<usize>,
-}
-
 #[cfg(not(feature = "private"))]
 pub struct AgentCache {}
 

backend/windmill-api/src/apps.rs

@@ -917,6 +917,23 @@ async fn create_app_internal<'a>(
     raw_app: bool,
     mut app: CreateApp,
 ) -> Result<(sqlx::Transaction<'a, sqlx::Postgres>, String, i64)> {
+    if *CLOUD_HOSTED {
+        let nb_apps =
+            sqlx::query_scalar!("SELECT COUNT(*) FROM app WHERE workspace_id = $1", &w_id)
+                .fetch_one(&db)
+                .await?;
+        if nb_apps.unwrap_or(0) >= 1000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of apps (1000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+        if app.summary.len() > 300 {
+            return Err(Error::BadRequest(
+                "Summary must be less than 300 characters on cloud".to_string(),
+            ));
+        }
+    }
     let mut tx = user_db.clone().begin(&authed).await?;
     app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
     app.policy.on_behalf_of_email = Some(authed.email.clone());
@@ -2011,7 +2028,7 @@ async fn upload_s3_file_from_app(
 
             if !has_unnamed_policy {
                 return Err(Error::BadRequest(
-                    "no policy found for unnamed s3 file uplooad".to_string(),
+                    "no policy found for unnamed s3 file upload".to_string(),
                 ));
             }
 

backend/windmill-api/src/configs.rs

@@ -21,7 +21,7 @@ use windmill_common::{
     DB,
 };
 
-use crate::{db::ApiAuthed, utils::require_super_admin};
+use crate::{db::ApiAuthed, utils::{require_devops_role}};
 
 pub fn global_service() -> Router {
     Router::new()
@@ -103,7 +103,7 @@ async fn get_config(
     Path(name): Path<String>,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Option<serde_json::Value>> {
-    require_super_admin(&db, &authed.email).await?;
+    require_devops_role(&db, &authed.email).await?;
 
     let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
         .fetch_optional(&db)
@@ -119,7 +119,7 @@ async fn update_config(
     authed: ApiAuthed,
     Json(config): Json<serde_json::Value>,
 ) -> error::Result<String> {
-    require_super_admin(&db, &authed.email).await?;
+    require_devops_role(&db, &authed.email).await?;
 
     #[cfg(not(feature = "enterprise"))]
     if name.starts_with("worker__") {
@@ -157,7 +157,7 @@ async fn delete_config(
     Extension(db): Extension<DB>,
     authed: ApiAuthed,
 ) -> error::Result<String> {
-    require_super_admin(&db, &authed.email).await?;
+    require_devops_role(&db, &authed.email).await?;
 
     let mut tx = db.begin().await?;
 
@@ -232,7 +232,7 @@ async fn list_configs(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Vec<Config>> {
-    require_super_admin(&db, &authed.email).await?;
+    require_devops_role(&db, &authed.email).await?;
     let configs = sqlx::query_as!(Config, "SELECT name, config FROM config")
         .fetch_all(&db)
         .await?;

backend/windmill-api/src/db.rs

@@ -812,6 +812,16 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
         .execute(db)
         .await?;
     });
+
+    run_windmill_migration!("audit_recent_login_activities", db, |tx| {
+        sqlx::query!(
+            "CREATE INDEX CONCURRENTLY idx_audit_recent_login_activities 
+ON audit (timestamp, username) 
+WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');"
+        )
+        .execute(db)
+        .await?;
+    });
     Ok(())
 }
 

backend/windmill-api/src/embeddings.rs

@@ -24,7 +24,7 @@ use candle_nn::VarBuilder;
 #[cfg(feature = "embedding")]
 use candle_transformers::models::bert::{BertModel, Config, DTYPE};
 #[cfg(feature = "embedding")]
-use hf_hub::{api::sync::Api, Cache, Repo};
+use hf_hub::api::tokio::Api;
 #[cfg(feature = "embedding")]
 use serde::Deserialize;
 #[cfg(feature = "embedding")]
@@ -158,63 +158,22 @@ pub struct ModelInstance {
 #[cfg(feature = "embedding")]
 impl ModelInstance {
     pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> {
-        let repo = Repo::model("thenlper/gte-small".to_string());
-
-        let cache = Cache::default().repo(repo.clone());
-
         let api = Api::new()?;
-        let api = api.repo(repo);
+        let repo_api = api.model("thenlper/gte-small".to_string());
 
-        let (config_filename, tokenizer_filename, weights_filename) = (
-            cache
-                .get("config.json")
-                .or_else(|| {
-                    api.get("config.json")
-                        .or_else(|e| {
-                            tracing::error!("Failed to get config.json from hugging face: {}", e);
-                            return Err(e);
-                        })
-                        .ok()
-                })
-                .ok_or(Error::msg("could not get config.json"))?,
-            cache
-                .get("tokenizer.json")
-                .or_else(|| {
-                    api.get("tokenizer.json")
-                        .or_else(|e| {
-                            tracing::error!(
-                                "Failed to get tokenizer.json from hugging face: {}",
-                                e
-                            );
-                            return Err(e);
-                        })
-                        .ok()
-                })
-                .ok_or(Error::msg("could not get tokenizer.json"))?,
-            cache
-                .get("model.safetensors")
-                .and_then(|p| {
-                    tracing::info!("Found embedding model in cache");
-                    Some(p)
-                })
-                .or_else(|| {
-                    tracing::info!("Downloading embedding model...");
-                    api.get("model.safetensors")
-                        .or_else(|e| {
-                            tracing::error!(
-                                "Failed to get model.safetensors from hugging face: {}",
-                                e
-                            );
-                            return Err(e);
-                        })
-                        .ok()
-                        .and_then(|p| {
-                            tracing::info!("Downloaded embedding model");
-                            Some(p)
-                        })
-                })
-                .ok_or(Error::msg("could not get model.safetensors"))?,
-        );
+        let (config_filename, tokenizer_filename, weights_filename) =
+            (
+                repo_api
+                    .get("config.json")
+                    .await
+                    .map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?,
+                repo_api.get("tokenizer.json").await.map_err(|e| {
+                    anyhow!("Failed to get tokenizer.json from hugging face: {}", e)
+                })?,
+                repo_api.get("model.safetensors").await.map_err(|e| {
+                    anyhow!("Failed to get model.safetensors from hugging face: {}", e)
+                })?,
+            );
 
         Ok((config_filename, tokenizer_filename, weights_filename))
     }

backend/windmill-api/src/flows.rs

@@ -34,7 +34,7 @@ use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::utils::query_elems_from_hub;
-use windmill_common::worker::to_raw_value;
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::HUB_BASE_URL;
 use windmill_common::{
     db::UserDB,
@@ -358,6 +358,32 @@ async fn create_flow(
     Path(w_id): Path<String>,
     Json(nf): Json<NewFlow>,
 ) -> Result<(StatusCode, String)> {
+    if *CLOUD_HOSTED {
+        let nb_flows =
+            sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id)
+                .fetch_one(&db)
+                .await?;
+        if nb_flows.unwrap_or(0) >= 1000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of flows (1000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+        if nf.summary.len() > 300 {
+            return Err(Error::BadRequest(
+                "Summary must be less than 300 characters on cloud".to_string(),
+            ));
+        }
+        if nf
+            .description
+            .as_ref()
+            .is_some_and(|desc| desc.len() > 3000)
+        {
+            return Err(Error::BadRequest(
+                "Description must be less than 3000 characters on cloud".to_string(),
+            ));
+        }
+    }
     #[cfg(not(feature = "enterprise"))]
     if nf
         .value

backend/windmill-api/src/gcp_triggers_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-api/src/gcp_triggers_ee.rs

backend/windmill-api/src/git_sync_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-api/src/git_sync_ee.rs

backend/windmill-api/src/inkeep_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-api/src/inkeep_ee.rs

backend/windmill-api/src/jobs.rs

@@ -3564,7 +3564,7 @@ pub async fn run_flow_by_path_inner(
         scheduled_for,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -3657,7 +3657,7 @@ pub async fn restart_flow(
         scheduled_for,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -3749,7 +3749,7 @@ pub async fn run_script_by_path_inner(
         scheduled_for,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -4417,7 +4417,7 @@ pub async fn run_wait_result_job_by_path_get(
         None,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -4557,7 +4557,7 @@ pub async fn run_wait_result_script_by_path_internal(
         None,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -4670,7 +4670,7 @@ pub async fn run_wait_result_script_by_hash(
         None,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -4784,7 +4784,7 @@ pub async fn run_wait_result_flow_by_path_internal(
         scheduled_for,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -5625,7 +5625,7 @@ pub async fn run_job_by_hash_inner(
         scheduled_for,
         None,
         run_query.parent_job,
-        run_query.root_job.or(run_query.parent_job),
+        run_query.root_job,
         run_query.job_id,
         false,
         false,
@@ -5875,6 +5875,7 @@ pub fn filter_list_completed_query(
         sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339()));
     }
     if let Some(dt) = &lq.created_or_started_after {
+        sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339()));
         sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339()));
     }
 

backend/windmill-api/src/lib.rs

@@ -36,8 +36,10 @@ use anyhow::Context;
 use argon2::Argon2;
 use axum::extract::DefaultBodyLimit;
 use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Router};
+use axum::response::Response;
+use axum::http::HeaderValue;
+use axum::body::Body;
 use db::DB;
-use http::HeaderValue;
 use reqwest::Client;
 #[cfg(feature = "oauth2")]
 use std::collections::HashMap;
@@ -236,6 +238,7 @@ lazy_static::lazy_static! {
 
 }
 
+
 // Compliance with cloud events spec.
 pub async fn add_webhook_allowed_origin(
     req: axum::extract::Request,
@@ -258,6 +261,7 @@ pub async fn add_webhook_allowed_origin(
     next.run(req).await
 }
 
+
 #[cfg(not(feature = "tantivy"))]
 type IndexReader = ();
 
@@ -890,12 +894,18 @@ async fn ee_license() -> String {
     }
 }
 
-async fn openapi() -> &'static str {
-    include_str!("../openapi-deref.yaml")
+async fn openapi() -> Response {
+    Response::builder()
+        .header("content-type", "application/yaml")
+        .body(Body::from(include_str!("../openapi-deref.yaml")))
+        .unwrap()
 }
 
-async fn openapi_json() -> &'static str {
-    include_str!("../openapi-deref.json")
+async fn openapi_json() -> Response {
+    Response::builder()
+        .header("content-type", "application/json")
+        .body(Body::from(include_str!("../openapi-deref.json")))
+        .unwrap()
 }
 
 pub async fn migrate_db(db: &DB) -> anyhow::Result<Option<JoinHandle<()>>> {

backend/windmill-api/src/mcp.rs

@@ -387,7 +387,8 @@ impl Runner {
         item_type: &str,
     ) -> Result<Vec<T>, Error> {
         let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type));
-        sqlb.fields(&["o.path", "o.summary", "o.description", "o.schema"]);
+        let fields = vec!["o.path", "o.summary", "o.description", "o.schema"];
+        sqlb.fields(&fields);
         if scope_type == "favorites" {
             sqlb.join("favorite")
                 .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type)
@@ -395,16 +396,21 @@ impl Runner {
         }
         sqlb.and_where("o.workspace_id = ?".bind(&workspace_id))
             .and_where("o.archived = false")
-            .and_where("o.draft_only IS NOT TRUE")
-            .order_by(
-                if item_type == "flow" {
-                    "o.edited_at"
-                } else {
-                    "o.created_at"
-                },
-                false,
-            )
-            .limit(100);
+            .and_where("o.draft_only IS NOT TRUE");
+
+        if item_type == "script" {
+            sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)");
+        }
+
+        sqlb.order_by(
+            if item_type == "flow" {
+                "o.edited_at"
+            } else {
+                "o.created_at"
+            },
+            false,
+        )
+        .limit(100);
         let sql = sqlb.sql().map_err(|_e| {
             tracing::error!("failed to build sql: {}", _e);
             Error::internal_error("failed to build sql", None)

backend/windmill-api/src/raw_apps.rs

@@ -29,6 +29,7 @@ use windmill_common::{
     db::UserDB,
     error::{Error, JsonResult, Result},
     utils::{not_found_if_none, paginate, Pagination, StripPath},
+    worker::CLOUD_HOSTED,
 };
 
 pub fn workspaced_service() -> Router {
@@ -149,9 +150,29 @@ async fn create_app(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
+    Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
+    if *CLOUD_HOSTED {
+        let nb_apps = sqlx::query_scalar!(
+            "SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_one(&db)
+        .await?;
+        if nb_apps.unwrap_or(0) >= 1000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of apps (1000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+        if app.summary.len() > 300 {
+            return Err(Error::BadRequest(
+                "Summary must be less than 300 characters on cloud".to_string(),
+            ));
+        }
+    }
     let mut tx = user_db.begin(&authed).await?;
     if &app.path == "" {
         return Err(Error::BadRequest("App path cannot be empty".to_string()));

backend/windmill-api/src/resources.rs

@@ -33,6 +33,7 @@ use windmill_common::{
     error::{Error, JsonResult, Result},
     utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
     variables,
+    worker::CLOUD_HOSTED,
 };
 
 pub fn workspaced_service() -> Router {
@@ -583,7 +584,6 @@ pub async fn transform_json_value<'c>(
                 job.schedule_path.clone(),
                 job.flow_step_id.clone(),
                 job.root_job.map(|x| x.to_string()),
-                None,
                 Some(job.scheduled_for.clone()),
             )
             .await;
@@ -657,6 +657,20 @@ async fn create_resource(
     Query(q): Query<CreateResourceQuery>,
     Json(resource): Json<CreateResource>,
 ) -> Result<(StatusCode, String)> {
+    if *CLOUD_HOSTED {
+        let nb_resources = sqlx::query_scalar!(
+            "SELECT COUNT(*) FROM resource WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_one(&db)
+        .await?;
+        if nb_resources.unwrap_or(0) >= 10000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of resources (10000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+    }
     let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await;
 
     let mut tx = user_db.begin(&authed).await?;

backend/windmill-api/src/scripts.rs

@@ -42,7 +42,7 @@ use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_worker::process_relative_imports;
 
-use windmill_common::error::to_anyhow;
+use windmill_common::{error::to_anyhow, worker::CLOUD_HOSTED};
 
 use windmill_common::{
     db::UserDB,
@@ -520,6 +520,29 @@ async fn create_script_internal<'c>(
                 .to_string(),
         ));
     }
+    if *CLOUD_HOSTED {
+        let nb_scripts =
+            sqlx::query_scalar!("SELECT COUNT(*) FROM script WHERE workspace_id = $1", &w_id)
+                .fetch_one(&db)
+                .await?;
+        if nb_scripts.unwrap_or(0) >= 5000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of scripts (5000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+
+        if ns.summary.len() > 300 {
+            return Err(Error::BadRequest(
+                "Summary must be less than 300 characters on cloud".to_string(),
+            ));
+        }
+        if ns.description.len() > 3000 {
+            return Err(Error::BadRequest(
+                "Description must be less than 3000 characters on cloud".to_string(),
+            ));
+        }
+    }
     let script_path = ns.path.clone();
     let hash = ScriptHash(hash_script(&ns));
     let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;

backend/windmill-api/src/static_assets.rs

@@ -10,6 +10,8 @@ use axum::{body::Body, extract::OriginalUri, http::Response, response::IntoRespo
 
 #[cfg(feature = "static_frontend")]
 use axum::http::header;
+#[cfg(feature = "static_frontend")]
+use http::HeaderValue;
 
 use hyper::Uri;
 #[cfg(feature = "static_frontend")]
@@ -17,6 +19,12 @@ use mime_guess::mime;
 #[cfg(feature = "static_frontend")]
 use rust_embed::RustEmbed;
 
+// Content Security Policy configuration  
+#[cfg(feature = "static_frontend")]
+lazy_static::lazy_static! {
+    static ref CSP_POLICY: String = std::env::var("CSP_POLICY").unwrap_or_default();
+}
+
 // static_handler is a handler that serves static files from the
 pub async fn static_handler(OriginalUri(original_uri): OriginalUri) -> StaticFile {
     StaticFile(original_uri)
@@ -51,6 +59,13 @@ fn serve_path(path: &str) -> Response<Body> {
             let mut res = Response::builder()
                 .header(header::CONTENT_TYPE, mime.as_ref())
                 .header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+            
+            // Add Content-Security-Policy header for static assets when policy is set
+            if !CSP_POLICY.is_empty() {
+                if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) {
+                    res = res.header("Content-Security-Policy", header_value);
+                }
+            }
             if mime.as_ref() == mime::APPLICATION_JAVASCRIPT
                 || mime.as_ref() == mime::TEXT_JAVASCRIPT
                 || path.ends_with(".wasm")

backend/windmill-api/src/teams_approvals_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-api/src/teams_approvals_ee.rs

backend/windmill-api/src/users.rs

@@ -1853,6 +1853,18 @@ async fn create_token(
     .fetch_optional(&mut *tx)
     .await?
     .unwrap_or(false);
+    if *CLOUD_HOSTED {
+        let nb_tokens =
+            sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email)
+                .fetch_one(&db)
+                .await?;
+        if nb_tokens.unwrap_or(0) >= 10000 {
+            return Err(Error::BadRequest(
+                "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit"
+                    .to_string(),
+            ));
+        }
+    }
     sqlx::query!(
         "INSERT INTO token
             (token, email, label, expiration, super_admin, scopes, workspace_id)

backend/windmill-api/src/utils.rs

@@ -415,3 +415,9 @@ pub struct ExpiringCacheEntry<T> {
     pub value: T,
     pub expiry: std::time::Instant,
 }
+
+#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))]
+pub async fn update_rw_lock<T>(lock: std::sync::Arc<tokio::sync::RwLock<T>>, value: T) -> () {
+    let mut w = lock.write().await;
+    *w = value;
+}

backend/windmill-api/src/variables.rs

@@ -29,6 +29,7 @@ use windmill_common::{
     variables::{
         build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable,
     },
+    worker::CLOUD_HOSTED,
 };
 
 use lazy_static::lazy_static;
@@ -74,8 +75,7 @@ async fn list_contextual_variables(
             Some("u/user/triggering_flow_path".to_string()),
             Some("c".to_string()),
             Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()),
-            Some("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c".to_string()),
-            Some(chrono::offset::Utc::now())
+            Some(chrono::offset::Utc::now()),
         )
         .await
         .to_vec(),
@@ -314,6 +314,20 @@ async fn create_variable(
     Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
     Json(variable): Json<CreateVariable>,
 ) -> Result<(StatusCode, String)> {
+    if *CLOUD_HOSTED {
+        let nb_variables = sqlx::query_scalar!(
+            "SELECT COUNT(*) FROM variable WHERE workspace_id = $1",
+            &w_id
+        )
+        .fetch_one(&db)
+        .await?;
+        if nb_variables.unwrap_or(0) >= 10000 {
+            return Err(Error::BadRequest(
+                    "You have reached the maximum number of variables (10000) on cloud. Contact support@windmill.dev to increase the limit"
+                        .to_string(),
+                ));
+        }
+    }
     let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await;
 
     check_path_conflict(&db, &w_id, &variable.path).await?;

backend/windmill-api/src/workspaces.rs

@@ -36,7 +36,7 @@ use windmill_common::db::UserDB;
 use windmill_common::s3_helpers::LargeFileStorage;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::variables::{build_crypt, decrypt, encrypt};
-use windmill_common::worker::to_raw_value;
+use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 #[cfg(feature = "enterprise")]
 use windmill_common::workspaces::WorkspaceDeploymentUISettings;
 #[cfg(feature = "enterprise")]
@@ -1454,6 +1454,21 @@ async fn create_workspace(
     #[cfg(not(feature = "enterprise"))]
     _check_nb_of_workspaces(&db).await?;
 
+    if *CLOUD_HOSTED {
+        let nb_workspaces = sqlx::query_scalar!(
+            "SELECT COUNT(*) FROM workspace WHERE owner = $1",
+            authed.email
+        )
+        .fetch_one(&db)
+        .await?;
+        if nb_workspaces.unwrap_or(0) >= 10 {
+            return Err(Error::BadRequest(
+                "You have reached the maximum number of workspaces (10) on cloud. Contact support@windmill.dev to increase the limit"
+                    .to_string(),
+            ));
+        }
+    }
+
     let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
     check_w_id_conflict(&mut tx, &nw.id).await?;

backend/windmill-api/src/workspaces_extra.rs

@@ -14,6 +14,7 @@ use windmill_audit::ActionKind;
 use windmill_common::worker::CLOUD_HOSTED;
 
 use windmill_common::{
+    auth::is_super_admin_email,
     error::{Error, Result},
     utils::require_admin,
 };
@@ -32,7 +33,7 @@ pub(crate) async fn change_workspace_id(
     Extension(db): Extension<DB>,
     Json(rw): Json<ChangeWorkspaceId>,
 ) -> Result<String> {
-    if *CLOUD_HOSTED {
+    if *CLOUD_HOSTED && !is_super_admin_email(&db, &authed.email).await? {
         return Err(Error::BadRequest(
             "This feature is not available on the cloud".to_string(),
         ));

backend/windmill-common/src/jobs.rs

@@ -159,7 +159,7 @@ impl QueuedJob {
 impl Default for QueuedJob {
     fn default() -> Self {
         Self {
-            workspace_id: "".to_string(),
+            workspace_id: "w".to_string(),
             id: Uuid::default(),
             parent_job: None,
             created_by: "".to_string(),

backend/windmill-common/src/oidc_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-common/src/oidc_ee.rs

backend/windmill-common/src/variables.rs

@@ -180,7 +180,6 @@ pub async fn get_reserved_variables(
     schedule_path: Option<String>,
     step_id: Option<String>,
     root_flow_id: Option<String>,
-    jwt_token: Option<String>,
     scheduled_for: Option<chrono::DateTime<Utc>>,
 ) -> Vec<ContextualVariable> {
     let state_path = {
@@ -334,12 +333,6 @@ pub async fn get_reserved_variables(
         description: "Script or flow step execution unique path, useful for storing results in an external service".to_string(),
         is_custom: false,
     },
-    ContextualVariable {
-        name: "WM_OIDC_JWT".to_string(),
-        value: jwt_token.unwrap_or_else(|| "".to_string()),
-        description: "OIDC JWT token (EE only)".to_string(),
-        is_custom: false,
-    },
     ContextualVariable {
         name: "WM_WORKER_GROUP".to_string(),
         value: WORKER_GROUP.clone(),

backend/windmill-queue/src/jobs.rs

@@ -2721,10 +2721,9 @@ pub async fn get_result_by_id(
                 "SELECT
                     id As \"id!\",
                     flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json<Uuid>\"
-                FROM v2_as_queue
-                WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id AND workspace_id = $2",
-                flow_id,
-                &w_id
+                FROM v2_job_status
+                WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id",
+                flow_id
             )
             .fetch_optional(&db)
             .await?;
@@ -2863,9 +2862,10 @@ pub async fn get_result_by_id_from_running_flow_inner(
     node_id: &str,
 ) -> error::Result<JobResult> {
     let flow_job_result = sqlx::query!(
-        "SELECT leaf_jobs->$1::text AS \"leaf_jobs: Json<Box<RawValue>>\", parent_job
-        FROM v2_as_queue
-        WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = id AND workspace_id = $3",
+        "SELECT flow_leaf_jobs->$1::text AS \"leaf_jobs: Json<Box<RawValue>>\", v2_job.parent_job
+        FROM v2_job_status
+        LEFT JOIN v2_job ON v2_job.id = v2_job_status.id AND v2_job.workspace_id = $3
+        WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = v2_job_status.id",
         node_id,
         flow_id,
         w_id,
@@ -2873,11 +2873,13 @@ pub async fn get_result_by_id_from_running_flow_inner(
     .fetch_optional(db)
     .await?;
 
+    // tracing::error!("flow_job_result: {:?} {:?}", flow_job_result, flow_id);
     let flow_job_result = windmill_common::utils::not_found_if_none(
         flow_job_result,
         "Root job of parent runnnig flow",
         format!("parent: {}, id: {}", flow_id, node_id),
     )?;
+    // tracing::error!("flow_job_result: {:?}, {:?}", flow_job_result.leaf_jobs, flow_job_result.parent_job);
 
     let job_result = flow_job_result
         .leaf_jobs
@@ -4028,6 +4030,7 @@ pub async fn push<'c, 'd>(
         ),
     };
 
+
     let final_priority: Option<i16>;
     #[cfg(not(feature = "enterprise"))]
     {
@@ -4229,10 +4232,10 @@ pub async fn push<'c, 'd>(
             INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,
                 created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,
             script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,
-            flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,
+            flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,
             cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
-            $19, $20, $21, $22, $23, $24, $25, $26,
+            $19, $20, $38, $21, $22, $23, $24, $25, $26,
             CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,
             ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)
         ),
@@ -4288,6 +4291,7 @@ pub async fn push<'c, 'd>(
         job_authed.is_operator,
         folders.as_slice(),
         job_authed.groups.as_slice(),
+        root_job.or(parent_job)
     )
     .execute(&mut *tx)
     .warn_after_seconds(1)

backend/windmill-queue/src/jobs_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-queue/src/jobs_ee.rs

backend/windmill-worker/src/bash_executor.rs

@@ -510,6 +510,23 @@ fn raw_to_string(x: &str) -> String {
         _ => String::new(),
     }
 }
+
+const POWERSHELL_INSTALL_CODE: &str = r#"
+$availableModules = Get-Module -ListAvailable
+$path = '{path}'
+
+$moduleNames = @({modules})
+
+foreach ($module in $moduleNames) {
+    if (-not ($availableModules | Where-Object { $_.Name -eq $module })) {
+        Write-Host "Installing module $module..."
+        Save-Module -Name $module -Path $path -Force
+    } else {
+        Write-Host "Module $module already installed"
+    }
+}
+"#;
+
 #[tracing::instrument(level = "trace", skip_all)]
 pub async fn handle_powershell_job(
     mem_peak: &mut i32,
@@ -573,27 +590,34 @@ pub async fn handle_powershell_job(
         })
         .collect::<Vec<String>>();
 
-    let mut install_string: String = String::new();
+    let mut modules_to_install: Vec<String> = Vec::new();
     let mut logs1 = String::new();
     for line in content.lines() {
         for cap in RE_POWERSHELL_IMPORTS.captures_iter(line) {
             let module = cap.get(1).unwrap().as_str();
             if !installed_modules.contains(&module.to_lowercase()) {
-                logs1.push_str(&format!("\n{} not found in cache", module.to_string()));
-                // instead of using Install-Module, we use Save-Module so that we can specify the installation path
-                install_string.push_str(&format!(
-                    "Save-Module -Path {} -Force {};",
-                    POWERSHELL_CACHE_DIR, module
-                ));
+                modules_to_install.push(module.to_string());
             } else {
                 logs1.push_str(&format!("\n{} found in cache", module.to_string()));
             }
         }
     }
 
-    if !install_string.is_empty() {
-        logs1.push_str("\n\nInstalling modules...");
+    if !logs1.is_empty() {
         append_logs(&job.id, &job.workspace_id, logs1, db).await;
+    }
+
+    if !modules_to_install.is_empty() {
+        let install_string = POWERSHELL_INSTALL_CODE
+            .replace("{path}", POWERSHELL_CACHE_DIR)
+            .replace(
+                "{modules}",
+                &modules_to_install
+                    .iter()
+                    .map(|x| format!("'{x}'"))
+                    .collect::<Vec<_>>()
+                    .join(", "),
+            );
         let child = Command::new(POWERSHELL_PATH.as_str())
             .args(&["-Command", &install_string])
             .stdout(Stdio::piped())

backend/windmill-worker/src/bun_executor.rs

@@ -1566,7 +1566,6 @@ pub async fn start_worker(
         None,
         None,
         None,
-        None,
     )
     .await;
     let context_envs = build_envs_map(context.to_vec()).await;

backend/windmill-worker/src/common.rs

@@ -434,7 +434,6 @@ pub async fn get_reserved_variables(
         job.schedule_path(),
         job.flow_step_id.clone(),
         job.flow_innermost_root_job.clone().map(|x| x.to_string()),
-        None,
         Some(job.scheduled_for.clone()),
     )
     .await

backend/windmill-worker/src/deno_executor.rs

@@ -11,8 +11,7 @@ use crate::{
         start_child_process, OccupancyMetrics,
     },
     handle_child::handle_child,
-    DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY,
-    PATH_ENV, TZ_ENV,
+    DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
 };
 use windmill_common::client::AuthedClient;
 
@@ -534,7 +533,6 @@ pub async fn start_worker(
         None,
         None,
         None,
-        None,
     )
     .await;
     let context_envs = build_envs_map(context.to_vec()).await;

backend/windmill-worker/src/otel_ee.rs

@@ -0,0 +1 @@
+/home/rubenfiszel/windmill/../windmill-ee-private/windmill-worker/src/otel_ee.rs

backend/windmill-worker/src/python_executor.rs

@@ -62,12 +62,52 @@ lazy_static::lazy_static! {
     static ref EPHEMERAL_TOKEN_CMD: Option<String> = var("EPHEMERAL_TOKEN_CMD").ok();
 }
 
+#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
+lazy_static::lazy_static! {
+    static ref PIPTAR_UPLOAD_CHANNEL: tokio::sync::mpsc::UnboundedSender<PiptarUploadTask> = {
+        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
+
+        // Spawn background task to handle uploads sequentially
+        tokio::spawn(handle_piptar_uploads(rx));
+
+        tx
+    };
+}
+
+#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
+#[derive(Debug)]
+struct PiptarUploadTask {
+    venv_path: String,
+    cache_dir: String,
+}
+
+#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
+async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver<PiptarUploadTask>) {
+    use crate::global_cache::build_tar_and_push;
+    use windmill_common::s3_helpers::get_object_store;
+
+    while let Some(task) = rx.recv().await {
+        if let Some(os) = get_object_store().await {
+            match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await {
+                Ok(()) => {
+                    tracing::info!("Successfully uploaded piptar for {}", task.venv_path);
+                }
+                Err(e) => {
+                    tracing::error!("Failed to upload piptar for {}: {}", task.venv_path, e);
+                }
+            }
+        } else {
+            tracing::warn!("S3 object store not available for piptar upload: {}", task.venv_path);
+        }
+    }
+}
+
 const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
 const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
 const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
 
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
-use crate::global_cache::{build_tar_and_push, pull_from_tar};
+use crate::global_cache::pull_from_tar;
 
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
 use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
@@ -284,6 +324,7 @@ pub async fn uv_pip_compile(
             child_cmd
                 .env("SystemRoot", SYSTEM_ROOT.as_str())
                 .env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
+                .env("HOME", crate::USERPROFILE_ENV.as_str())
                 .env(
                     "LOCALAPPDATA",
                     std::env::var("LOCALAPPDATA")
@@ -292,6 +333,29 @@ pub async fn uv_pip_compile(
                 .env(
                     "TMP",
                     std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
+                )
+                .env(
+                    "APPDATA",
+                    std::env::var("APPDATA")
+                        .unwrap_or_else(|_| format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())),
+                )
+                .env(
+                    "ComSpec",
+                    std::env::var("ComSpec").unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
+                )
+                .env(
+                    "PATHEXT",
+                    std::env::var("PATHEXT").unwrap_or_else(|_|
+                        String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
+                    ),
+                )
+                .env(
+                    "ProgramData",
+                    std::env::var("ProgramData").unwrap_or_else(|_| String::from("C:\\ProgramData")),
+                )
+                .env(
+                    "ProgramFiles",
+                    std::env::var("ProgramFiles").unwrap_or_else(|_| String::from("C:\\Program Files")),
                 );
         }
 
@@ -562,7 +626,7 @@ pub async fn handle_python_job(
             if v == '<function call>':
                 del pre_args[k]
         kwargs = inner_script.preprocessor(**pre_args)
-        kwrags_json = res_to_json(kwargs)    
+        kwrags_json = res_to_json(kwargs)
         with open("args.json", 'w') as f:
             f.write(kwrags_json)"#
         )
@@ -638,7 +702,7 @@ except BaseException as e:
     tb = traceback.format_tb(exc_traceback)
     with open(result_json, 'w') as f:
         err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}
-        extra = e.__dict__ 
+        extra = e.__dict__
         if extra and len(extra) > 0:
             err['extra'] = extra
         flow_node_id = os.environ.get('WM_FLOW_STEP_ID')
@@ -674,7 +738,14 @@ except BaseException as e:
             //    ^^^^^^ ^
             // We also want this be priorotized, that's why we insert it to the beginning
         }
-        paths.iter().join(":")
+        #[cfg(windows)]
+        {
+            paths.iter().join(";")
+        }
+        #[cfg(not(windows))]
+        {
+            paths.iter().join(":")
+        }
     };
 
     #[cfg(windows)]
@@ -1370,6 +1441,7 @@ async fn spawn_uv_install(
                 .envs(PROXY_ENVS.clone())
                 .env("SystemRoot", SYSTEM_ROOT.as_str())
                 .env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
+                .env("HOME", HOME_ENV.as_str())
                 .env(
                     "TMP",
                     std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
@@ -1379,6 +1451,29 @@ async fn spawn_uv_install(
                     std::env::var("LOCALAPPDATA")
                         .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
                 )
+                .env(
+                    "APPDATA",
+                    std::env::var("APPDATA")
+                        .unwrap_or_else(|_| format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())),
+                )
+                .env(
+                    "ComSpec",
+                    std::env::var("ComSpec").unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
+                )
+                .env(
+                    "PATHEXT",
+                    std::env::var("PATHEXT").unwrap_or_else(|_|
+                        String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
+                    ),
+                )
+                .env(
+                    "ProgramData",
+                    std::env::var("ProgramData").unwrap_or_else(|_| String::from("C:\\ProgramData")),
+                )
+                .env(
+                    "ProgramFiles",
+                    std::env::var("ProgramFiles").unwrap_or_else(|_| String::from("C:\\Program Files")),
+                )
                 .args(&command_args[1..])
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped());
@@ -1763,7 +1858,7 @@ pub async fn handle_python_reqs(
 
                                 // Create a file to indicate that installation was successfull
                                 let valid_path = venv_p.clone() + "/.valid.windmill";
-                                // This is atomic operation, meaning, that it either completes and wheel is valid, 
+                                // This is atomic operation, meaning, that it either completes and wheel is valid,
                                 // or it does not and wheel is invalid and will be reinstalled next run
                                 if let Err(e) = File::create(&valid_path).await{
                                     tracing::error!(
@@ -1890,8 +1985,16 @@ pub async fn handle_python_reqs(
 
             #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
             if s3_push {
-                if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
-                    tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false));
+                // Send to upload channel for sequential processing
+                let upload_task = PiptarUploadTask {
+                    venv_path: venv_p.clone(),
+                    cache_dir: py_version.to_cache_dir_top_level(false),
+                };
+
+                if let Err(e) = PIPTAR_UPLOAD_CHANNEL.send(upload_task) {
+                    tracing::warn!("Failed to queue piptar upload for {venv_p}: {e}");
+                } else {
+                    tracing::info!("Queued piptar upload for {venv_p}");
                 }
             }
 
@@ -1906,7 +2009,7 @@ pub async fn handle_python_reqs(
             pids.lock().await.get_mut(i).and_then(|e| e.take());
             // Create a file to indicate that installation was successfull
             let valid_path = venv_p.clone() + "/.valid.windmill";
-            // This is atomic operation, meaning, that it either completes and wheel is valid, 
+            // This is atomic operation, meaning, that it either completes and wheel is valid,
             // or it does not and wheel is invalid and will be reinstalled next run
             if let Err(e) = File::create(&valid_path).await{
                 tracing::error!(
@@ -2022,7 +2125,6 @@ pub async fn start_worker(
         None,
         None,
         None,
-        None,
     )
     .await
     .to_vec();
@@ -2142,7 +2244,6 @@ for line in sys.stdin:
         None,
         None,
         None,
-        None,
     )
     .await;
 
@@ -2193,3 +2294,4 @@ for line in sys.stdin:
     )
     .await
 }
+