Merge branch 'main' into darkmode_new

* feat(frontend): wip

* feat(frontend): wip

* feat(frontend): implement cellRenderer

* feat(frontend): wip

* feat(frontend): wip

* feat(frontend): Fix actions width

* feat(frontend): Fix interactions

* feat(frontend): Fix styling

* feat(frontend): Correctly handleAgGrid EE

* feat(frontend): Fix select

* feat(frontend): Fix select

* feat(frontend): Fix how row is passed

* feat(frontend): Fix update

* feat(frontend): Correctly implement cache

* feat(frontend): Correctly implement cache

* feat(frontend): simplify how refreshActions work

* feat(frontend): improve comparaison

* feat(frontend): simplify refreshactins

* feat(frontend): clean up

* feat(frontend): clean up

* feat(frontend): add inputs output + fix row eval

* feat(frontend): add config to wrap actions

* feat(frontend): add config to wrap actions

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

.github/workflows/benchmark.yml

@@ -34,27 +34,20 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_single
+          path: |
+            *.json
 
   benchmark_dedicated:
-    needs: benchmark_single
     runs-on: ubicloud-standard-8
     services:
       postgres:
@@ -82,24 +75,18 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
           --no-warm-up -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_dedicated
+          path: |
+            *.json
 
   benchmark_4workers:
     runs-on: ubicloud-standard-8
@@ -162,9 +149,6 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
@@ -172,15 +156,12 @@ jobs:
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
           --workers 4
-      - name: Push changes
-        run: |
-          pwd
-          git pull origin benchmarks
-          git add .
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git commit -m "Update benchmarks"
-          git push
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_4workers
+          path: |
+            *.json
 
   benchmark_8workers:
     runs-on: ubicloud-standard-8
@@ -286,9 +267,6 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - uses: actions/checkout@v4
-        with:
-          ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
         run: deno run --unstable -A -r
@@ -296,10 +274,40 @@ jobs:
           -c
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
           --workers 8
+      - name: Save benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_8workers
+          path: |
+            *.json
+
+  benchmark_graphs:
+    runs-on: ubicloud
+    needs:
+      - benchmark_single
+      - benchmark_dedicated
+      - benchmark_4workers
+      - benchmark_8workers
+    steps:
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+      - uses: actions/checkout@v4
+        with:
+          ref: benchmarks
+      - name: Download benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+      - name: graphs
+        run: deno run --unstable -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json
       - name: Push changes
         run: |
+          ls -la
           pwd
-          git pull origin benchmarks
           git add .
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"

.github/workflows/build_ws.yml

@@ -7,15 +7,14 @@ name: Publish websocket multiplayer server
 on:
   workflow_dispatch:
 
-
 permissions:
   contents: read
   id-token: write
   packages: write
-  
+
 jobs:
   publish_multiplayer:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
         with:
@@ -56,7 +55,7 @@ jobs:
 
   publish_privately:
     needs: [publish_multiplayer]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
         with:
@@ -71,9 +70,8 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-
       - name: Push image to ECR
         run: |
           docker buildx imagetools create \
             --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.github/workflows/docker-image.yml

@@ -8,15 +8,16 @@ on:
   push:
     branches: [main]
     tags: ["*"]
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "Dockerfile"
 
 concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: write
-  id-token: write
-  packages: write
+permissions: write-all
 
 jobs:
   build:
@@ -136,7 +137,6 @@ jobs:
           push: true
           build-args: |
             features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect
-            nsjail=true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
             ${{ steps.meta-ee-public.outputs.tags }}
@@ -144,52 +144,68 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-  build_ee_reports_privately:
-    needs: [build_ee]
+  build_ee_312:
     runs-on: ubicloud
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v3
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: 0
+
       # - name: Set up Docker Buildx
       #   uses: docker/setup-buildx-action@v2
-
       - uses: depot/setup-action@v1
 
       - name: Docker meta
-        id: meta-ee-public
+        id: meta-ee-public-py312
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-py312
+          flavor: |
+            latest=false
           tags: |
-            type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
-            type=sha,enable=true,priority=100,prefix=,suffix=,format=short
 
-      - name: Login to ECR
-        if: github.event_name != 'pull_request'
+      - name: Login to registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.ECR_REGISTRY }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push publicly ee reports
+      - name: Substitute EE code
+        run: |
+          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+
+      - name: Build and push publicly ee
         uses: depot/build-push-action@v1
         with:
           context: .
           platforms: linux/amd64
           push: true
-          file: "./docker/DockerfileReports"
+          build-args: |
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect
+            PYTHON_IMAGE=python:3.12.2-slim-bookworm
           tags: |
-            ${{ steps.meta-ee-public.outputs.tags }}
+            ${{ steps.meta-ee-public-py312.outputs.tags }}
           labels: |
-            ${{ steps.meta-ee-public.outputs.labels }}
+            ${{ steps.meta-ee-public-py312.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
-
   # disabled until we make it 100% reliable and add more meaningful tests
   # playwright:
   #   runs-on: [self-hosted, new]
@@ -311,54 +327,6 @@ jobs:
   #           ${{ steps.extract.outputs.destination }}/*
   #           ${{ steps.extract-ee.outputs.destination }}/*
 
-  publish_ecr_s3:
-    needs: [build_ee]
-    runs-on: ubicloud
-    if: github.event_name != 'pull_request'
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to ECR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.ECR_REGISTRY }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-      - name: Push image to ECR
-        if: github.event_name != 'pull_request'
-        id: push_ecr
-        run: |
-          git_hash=$(git rev-parse --short "$GITHUB_SHA")
-          docker buildx imagetools create \
-            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
-          echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"
-
-      - uses: shrink/actions-docker-extract@v3
-        if: github.event_name != 'pull_request'
-        id: extract
-        with:
-          image: |-
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ steps.push_ecr.outputs.GIT_HASH }}
-          path: "/static_frontend/."
-
-      - uses: reggionick/s3-deploy@v3
-        if: github.event_name != 'pull_request'
-        with:
-          folder: ${{ steps.extract.outputs.destination }}
-          bucket: windmill-frontend
-          bucket-region: us-east-1
-
   run_integration_test:
     runs-on: ubicloud
     needs: [build_ee]
@@ -419,6 +387,190 @@ jobs:
           docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
           docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
 
+  verify_ee_image_vulnerabilities:
+    runs-on: ubicloud
+    needs: [tag_latest_ee]
+    # if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Analyze for critical and high CVEs
+        id: docker-scout-cves
+        if: ${{ github.event_name != 'pull_request_target' }}
+        uses: docker/scout-action@v1
+        with:
+          command: cves
+          only-severities: critical,high
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
+          sarif-file: sarif.output.json
+          summary: true
+          dockerhub-user: windmilllabs
+          dockerhub-password: ${{ secrets.DOCKER_PAT }}
+
+      - name: Upload SARIF result
+        id: upload-sarif
+        if: ${{ github.event_name != 'pull_request_target' }}
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: sarif.output.json
+
+  # docker_scout_ee:
+  #   runs-on: ubicloud
+  #   needs: [tag_latest_ee]
+  #   steps:
+  #     - name: Docker Scout
+  #       id: docker-scout
+  #       uses: docker/scout-action@v1
+  #       with:
+  #         dockerhub-
+  #         command: cves,recommendations,compare
+  #         to-latest: true
+  #         ignore-base: true
+  #         ignore-unchanged: true
+  #         only-fixed: true
+
+  build_ee_nsjail:
+    needs: [build_ee]
+    runs-on: ubicloud
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Docker meta
+        id: meta-ee-public
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=short
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          file: "./docker/DockerfileNsjail"
+          tags: |
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+
+  build_ee_reports_privately:
+    needs: [build_ee_nsjail]
+    runs-on: ubicloud
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Docker meta
+        id: meta-ee-public
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=short
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Build and push publicly ee reports
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          file: "./docker/DockerfileReports"
+          tags: |
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+
+  publish_ecr_s3:
+    needs: [build_ee_nsjail]
+    runs-on: ubicloud
+    if: github.event_name != 'pull_request'
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Push image to ECR
+        if: github.event_name != 'pull_request'
+        id: push_ecr
+        run: |
+          git_hash=$(git rev-parse --short "$GITHUB_SHA")
+          docker buildx imagetools create \
+            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail:main
+          echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"
+
+      - uses: shrink/actions-docker-extract@v3
+        if: github.event_name != 'pull_request'
+        id: extract
+        with:
+          image: |-
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ steps.push_ecr.outputs.GIT_HASH }}
+          path: "/static_frontend/."
+
+      - uses: reggionick/s3-deploy@v3
+        if: github.event_name != 'pull_request'
+        with:
+          folder: ${{ steps.extract.outputs.destination }}
+          bucket: windmill-frontend
+          bucket-region: us-east-1
+
   build_ee_cuda:
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
     needs: [build_ee]

.github/workflows/release-please.yml

@@ -6,7 +6,7 @@ name: release-please
 jobs:
   release-please:
     name: "Release please"
-    runs-on: ubicloud-standard-8
+    runs-on: ubicloud
     steps:
       - uses: GoogleCloudPlatform/release-please-action@v3
         with:

.github/workflows/sign-cla.yml

@@ -7,10 +7,13 @@ on:
 
 jobs:
   CLAssistant:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     steps:
       - name: "CLA Assistant"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        if:
+          (github.event.comment.body == 'recheck' || github.event.comment.body
+          == 'I have read the CLA Document and I hereby sign the CLA') ||
+          github.event_name == 'pull_request_target'
         # Beta Release
         uses: cla-assistant/github-action@v2.3.1
         env:

CHANGELOG.md

@@ -1,5 +1,249 @@
 # Changelog
 
+## [1.305.3](https://github.com/windmill-labs/windmill/compare/v1.305.2...v1.305.3) (2024-04-12)
+
+
+### Bug Fixes
+
+* improve app icon renderer and app html renderer ([ee56821](https://github.com/windmill-labs/windmill/commit/ee56821a4734fa56097ae1ec7e032b6d14c0ee80))
+
+## [1.305.2](https://github.com/windmill-labs/windmill/compare/v1.305.1...v1.305.2) (2024-04-11)
+
+
+### Bug Fixes
+
+* unsaved changes on deploy of flow ([d1650db](https://github.com/windmill-labs/windmill/commit/d1650dbed935b206a8de8a59835933d559ff3a20))
+
+## [1.305.1](https://github.com/windmill-labs/windmill/compare/v1.305.0...v1.305.1) (2024-04-11)
+
+
+### Bug Fixes
+
+* fix plug connection for apps with array items ([c88e6a8](https://github.com/windmill-labs/windmill/commit/c88e6a86359cbdad038db09d54979cec1f12c8f3))
+* **frontend:** add missing darkModeObserver in CronInput ([#3531](https://github.com/windmill-labs/windmill/issues/3531)) ([8525494](https://github.com/windmill-labs/windmill/commit/8525494a68314e342b7393f6b6df15565ffb4082))
+* **frontend:** fix text alignement ([#3533](https://github.com/windmill-labs/windmill/issues/3533)) ([0c9a95c](https://github.com/windmill-labs/windmill/commit/0c9a95c964a16c3194d4887b7a3882a626faf6e4))
+* improve goto behavior ([6c33f17](https://github.com/windmill-labs/windmill/commit/6c33f17829a3a43cd1794abfc0c0807caa01bcfd))
+* load lazily the app icons ([cd5b023](https://github.com/windmill-labs/windmill/commit/cd5b0230338f3f20d7306b8a3feef509f6c8fe30))
+* remove requirement on full wasm parser for row insert of db studio ([cd07020](https://github.com/windmill-labs/windmill/commit/cd07020edb8ff652d086023771caa07755b5b2a2))
+* remove requirement on full wasm parser for row insert of db studio ([f17bed9](https://github.com/windmill-labs/windmill/commit/f17bed9741762f5241206a1bc257bcc4c4cb96fb))
+* update monaco configurations ([62e4ec2](https://github.com/windmill-labs/windmill/commit/62e4ec2e1443b47c273631d2b8efa7f3a9ba804c))
+
+## [1.305.0](https://github.com/windmill-labs/windmill/compare/v1.304.4...v1.305.0) (2024-04-09)
+
+
+### Features
+
+* flow concurrency limits support custom concurrency key ([a55aad3](https://github.com/windmill-labs/windmill/commit/a55aad30039d8bc44f4cbc9aa2fa3e220966ca80))
+
+
+### Bug Fixes
+
+* add ability to cancel flows directly from operator modal ([5272956](https://github.com/windmill-labs/windmill/commit/5272956153419f3eac7203a9b043ca63c2cf869b))
+
+## [1.304.4](https://github.com/windmill-labs/windmill/compare/v1.304.3...v1.304.4) (2024-04-09)
+
+
+### Bug Fixes
+
+* allow for longer approver names in flows ([f117539](https://github.com/windmill-labs/windmill/commit/f117539a54e5947832c1d435a4991274d103cef0))
+* **frontend:** Fix bypass confirmation shortcut ([#3527](https://github.com/windmill-labs/windmill/issues/3527)) ([db5abd3](https://github.com/windmill-labs/windmill/commit/db5abd37cda548e642fa825cd279a730a7af5c00))
+* **frontend:** Improve theme editor ([#3525](https://github.com/windmill-labs/windmill/issues/3525)) ([975055b](https://github.com/windmill-labs/windmill/commit/975055b90ff3e7bd0b8202920ca78fb9178663a7))
+* improve deadlocks for parallel branches with parallelism constraints ([0c824de](https://github.com/windmill-labs/windmill/commit/0c824de4cdfb3b6b609cce0e1678a58743ea24f4))
+* improve deadlocks for parallel branches with parallelism constraints ([8dd1175](https://github.com/windmill-labs/windmill/commit/8dd117528c793b828631d5f320bb28ce9c5e3a3f))
+* improve handling of very large iterator on frontend ([7e4b6c3](https://github.com/windmill-labs/windmill/commit/7e4b6c374415862954f4876b693a199aabcab45e))
+* improve order dragndrop for json editor in app ([1ab1fb7](https://github.com/windmill-labs/windmill/commit/1ab1fb7e4de9984c6408d1fe5ef332d8f242362b))
+* improve performance of for-loops with parralelism ([a4442d4](https://github.com/windmill-labs/windmill/commit/a4442d4d3d919383ac09df66f20d6c4f98c04a92))
+* improve performance of for-loops with parralelism ([29422f1](https://github.com/windmill-labs/windmill/commit/29422f156d8ca3e0ac03288bd11b31f505a492c1))
+
+## [1.304.3](https://github.com/windmill-labs/windmill/compare/v1.304.2...v1.304.3) (2024-04-08)
+
+
+### Bug Fixes
+
+* add resource types to list of ignored path filters for git sync ([fc0056f](https://github.com/windmill-labs/windmill/commit/fc0056f991d985e3a014ac50a7fe2ef73bb6b97a))
+* add resource types to list of ignored path filters for git sync ([acf80dc](https://github.com/windmill-labs/windmill/commit/acf80dc6a005ccd048ffc193841b3c599e93c50d))
+* improve CLI with visible_to_runner_only and priority ([c7f0f3f](https://github.com/windmill-labs/windmill/commit/c7f0f3f359ff57ea1f816dc5598a61d50c3ae263))
+
+## [1.304.2](https://github.com/windmill-labs/windmill/compare/v1.304.1...v1.304.2) (2024-04-06)
+
+
+### Bug Fixes
+
+* **frontend:** Fix selectFirstRowByDefault on AgGrids ([#3450](https://github.com/windmill-labs/windmill/issues/3450)) ([a52d6b3](https://github.com/windmill-labs/windmill/commit/a52d6b30b21bbf8d42cef1cb760b082181163fa2))
+
+## [1.304.1](https://github.com/windmill-labs/windmill/compare/v1.304.0...v1.304.1) (2024-04-06)
+
+
+### Bug Fixes
+
+* remove reqwest_11 ([8cdae1a](https://github.com/windmill-labs/windmill/commit/8cdae1ac965f8bbb03fff1fb68b2d2ee3c1f744f))
+
+## [1.304.0](https://github.com/windmill-labs/windmill/compare/v1.303.4...v1.304.0) (2024-04-06)
+
+
+### Features
+
+* add overridable cache_ttl in api of job triggers ([21a7ee4](https://github.com/windmill-labs/windmill/commit/21a7ee4c33c483d39cc6d7020623ee11fc9f1cbd))
+* git sync group changes by folder ([#3517](https://github.com/windmill-labs/windmill/issues/3517)) ([08231c0](https://github.com/windmill-labs/windmill/commit/08231c02d282410a7939e358a49cad2f4696dacd))
+
+
+### Bug Fixes
+
+* add args filter to schedule list ([f9d8dde](https://github.com/windmill-labs/windmill/commit/f9d8dde61cc527d5f9553f3981485a8c345dc1aa))
+* fix lightweight arg input min size ([78b8c3e](https://github.com/windmill-labs/windmill/commit/78b8c3edc9e28a07d8557c0d34208f773539fa72))
+* **frontend:** Fix simple flow tutorial ([#3518](https://github.com/windmill-labs/windmill/issues/3518)) ([139bc38](https://github.com/windmill-labs/windmill/commit/139bc38ddcc0988a1485f9d53535631626cab7fa))
+* improve list static input ([d61d6f5](https://github.com/windmill-labs/windmill/commit/d61d6f520bd52df23f020cd12cbaf666cfe0097b))
+* make autodatatable more resilient ([fba1ea7](https://github.com/windmill-labs/windmill/commit/fba1ea7d547791cfa6dfd62b51db9ba567cfa243))
+* make autodatatable more resilient ([632c9fb](https://github.com/windmill-labs/windmill/commit/632c9fb2aab2fc073e5526b2c9f7d363bae063b4))
+* make date time input more resilient ([2ec1add](https://github.com/windmill-labs/windmill/commit/2ec1add494afa1a56c62ae430429912a095dd85c))
+* parallel flow with parallelism constraint could deadlock ([9131d5c](https://github.com/windmill-labs/windmill/commit/9131d5cc4088ca66383ca9e4e10fedae0e9e661f))
+* remove information in approval page of a flow ([3175456](https://github.com/windmill-labs/windmill/commit/31754569bae3355d58d20eea2749ad203a92a6dd))
+* show flow user states in flow state preview ([7c4eece](https://github.com/windmill-labs/windmill/commit/7c4eece009717e4188eeb8b1dedd094589dc2bdf))
+
+## [1.303.4](https://github.com/windmill-labs/windmill/compare/v1.303.3...v1.303.4) (2024-04-05)
+
+
+### Bug Fixes
+
+* prevent operators from loading secrets through toggle ([72f4247](https://github.com/windmill-labs/windmill/commit/72f424798386ab011e2d618a1010ef77a06435a1))
+
+## [1.303.3](https://github.com/windmill-labs/windmill/compare/v1.303.2...v1.303.3) (2024-04-04)
+
+
+### Bug Fixes
+
+* improve multiselect for approval steps ([18e7e94](https://github.com/windmill-labs/windmill/commit/18e7e9449e82582ddffc325ce8aeab88635cf7bc))
+
+## [1.303.2](https://github.com/windmill-labs/windmill/compare/v1.303.1...v1.303.2) (2024-04-04)
+
+
+### Bug Fixes
+
+* improve global cache deps ([0b3e6b9](https://github.com/windmill-labs/windmill/commit/0b3e6b9f1cb08265dec5573741f5911e09f79884))
+
+## [1.303.1](https://github.com/windmill-labs/windmill/compare/v1.303.0...v1.303.1) (2024-04-04)
+
+
+### Bug Fixes
+
+* improve chars splitting for logs with utf-8 chars ([2e3b6f6](https://github.com/windmill-labs/windmill/commit/2e3b6f66e3f22ea8e68531abb36fe0ca6424c766))
+
+## [1.303.0](https://github.com/windmill-labs/windmill/compare/v1.302.0...v1.303.0) (2024-04-04)
+
+
+### Features
+
+* private hub ([#3491](https://github.com/windmill-labs/windmill/issues/3491)) ([1d7bab0](https://github.com/windmill-labs/windmill/commit/1d7bab075bf4cfe614694575c3aba8c784f5f148))
+
+
+### Bug Fixes
+
+* update awscli & nodejs ([74e0f21](https://github.com/windmill-labs/windmill/commit/74e0f21903d858edcf8396930d51b7d29668cc02))
+
+## [1.302.0](https://github.com/windmill-labs/windmill/compare/v1.301.0...v1.302.0) (2024-04-04)
+
+
+### Features
+
+* slack team connected to multiple workspaces ([#3500](https://github.com/windmill-labs/windmill/issues/3500)) ([a8bf075](https://github.com/windmill-labs/windmill/commit/a8bf0750a475f16e7efd47e9a7fd120a19d52189))
+
+
+### Bug Fixes
+
+* build ([#3504](https://github.com/windmill-labs/windmill/issues/3504)) ([285aec4](https://github.com/windmill-labs/windmill/commit/285aec4ffdf00d6616e75749df7d080705bee54c))
+* fix char excess size module for logs truncation ([826757b](https://github.com/windmill-labs/windmill/commit/826757b33ab9ded921e98f06f0921bf3c9916e8b))
+* improve locking ([281cd7b](https://github.com/windmill-labs/windmill/commit/281cd7bc8b903b8809b840b8e55df0b962eef787))
+
+## [1.301.0](https://github.com/windmill-labs/windmill/compare/v1.300.0...v1.301.0) (2024-04-02)
+
+
+### Features
+
+* while loop as new flow primitive ([ff26c8e](https://github.com/windmill-labs/windmill/commit/ff26c8e42dbd212e75bb094cd34b8848d5e39fb6))
+
+
+### Bug Fixes
+
+* add access to the schedules page to operators ([648accd](https://github.com/windmill-labs/windmill/commit/648accd268b7ccbc27e3d6f1051540138ae5525e))
+* date-fns tooltip info again ([#3498](https://github.com/windmill-labs/windmill/issues/3498)) ([71d98b9](https://github.com/windmill-labs/windmill/commit/71d98b9025b622eddb894fd77d90f99f805a8bdc))
+
+## [1.300.0](https://github.com/windmill-labs/windmill/compare/v1.299.1...v1.300.0) (2024-04-02)
+
+
+### Features
+
+* **frontend:** Add an onRecompute callback to handle recompute side … ([#3493](https://github.com/windmill-labs/windmill/issues/3493)) ([3afa975](https://github.com/windmill-labs/windmill/commit/3afa97527090a82e1754db4d92322c940a532d35))
+
+
+### Bug Fixes
+
+* date-fns tooltip format info ([#3496](https://github.com/windmill-labs/windmill/issues/3496)) ([e4bca00](https://github.com/windmill-labs/windmill/commit/e4bca0021047db00c05a68a5238215ca66ba5a25))
+* **frontend:** Fix updateCellValue ([#3497](https://github.com/windmill-labs/windmill/issues/3497)) ([e4d8de8](https://github.com/windmill-labs/windmill/commit/e4d8de80ef9569bdfe06f8fbc93c8c6fa93e472c))
+* **typescript-client:** improve setFlowUserState undefined value ([ba679c6](https://github.com/windmill-labs/windmill/commit/ba679c64b7f15e42b8bede0d1b5cf03554049d6e))
+
+## [1.299.1](https://github.com/windmill-labs/windmill/compare/v1.299.0...v1.299.1) (2024-03-31)
+
+
+### Bug Fixes
+
+* fix audit issue with webhook triggered scripts ([4ba1f6c](https://github.com/windmill-labs/windmill/commit/4ba1f6ccdf62c3e778e1e1cbad1bdc3072bb8c17))
+
+## [1.299.0](https://github.com/windmill-labs/windmill/compare/v1.298.0...v1.299.0) (2024-03-30)
+
+
+### Features
+
+* add workspace free-tier usage ([#3489](https://github.com/windmill-labs/windmill/issues/3489)) ([b4ffb50](https://github.com/windmill-labs/windmill/commit/b4ffb500ba5c58932d9fe508186f8fbced3f3b71))
+* **frontend:** add a refresh component control to the DB Studio ([#3490](https://github.com/windmill-labs/windmill/issues/3490)) ([d713566](https://github.com/windmill-labs/windmill/commit/d71356695f981c5861ad9cb20223e1b096d1a34d))
+* **frontend:** Add support for clearFiles in the file input component ([#3483](https://github.com/windmill-labs/windmill/issues/3483)) ([f9a5bce](https://github.com/windmill-labs/windmill/commit/f9a5bcee4c0bf75b07ac7e15a19c420db81eb738))
+* support gh markdown for script, flows, resource descriptions ([6cb2d20](https://github.com/windmill-labs/windmill/commit/6cb2d20b423cfd9a4af072a561bf8583a54d153b))
+
+
+### Bug Fixes
+
+* benchmark action single push ([#3480](https://github.com/windmill-labs/windmill/issues/3480)) ([b3ed678](https://github.com/windmill-labs/windmill/commit/b3ed6782f612091af881dc5b49ca9bba814e5a26))
+* db studio multiple schemas support ([#3479](https://github.com/windmill-labs/windmill/issues/3479)) ([6489697](https://github.com/windmill-labs/windmill/commit/64896971bb17186b68a0f98e1a40eb1e585d3352))
+* **frontend:** Fix the pane delete button ([#3482](https://github.com/windmill-labs/windmill/issues/3482)) ([7b3b96e](https://github.com/windmill-labs/windmill/commit/7b3b96ea088720af96db0aa0da3395f44725181e))
+* make submit form order static ([951758f](https://github.com/windmill-labs/windmill/commit/951758ffb631b313d0f033bb40dbe4f16383dfb7))
+* show script path when schedule summary is empty ([#3487](https://github.com/windmill-labs/windmill/issues/3487)) ([3c38491](https://github.com/windmill-labs/windmill/commit/3c384910c16a7bfa3c8c9aade8ca8f602ac093b8))
+* show start to finish time for flows instead of cumulative ([#3486](https://github.com/windmill-labs/windmill/issues/3486)) ([e3a636a](https://github.com/windmill-labs/windmill/commit/e3a636ab178ea6d1d6a49040325081ebb23fa3e7))
+* tree view show more ([a61b14c](https://github.com/windmill-labs/windmill/commit/a61b14cfd40544b2934f8472bdd82c4242bda1bc))
+
+## [1.298.0](https://github.com/windmill-labs/windmill/compare/v1.297.1...v1.298.0) (2024-03-27)
+
+
+### Features
+
+* add map support in renderer ([240ae93](https://github.com/windmill-labs/windmill/commit/240ae9374885fbc9a679d6c3d18a58f08c344756))
+* approval steps description ([810136a](https://github.com/windmill-labs/windmill/commit/810136a4a405855801c94de66ec8916979701c67))
+
+
+### Bug Fixes
+
+* display approval step at top level regardless of depth ([57a0530](https://github.com/windmill-labs/windmill/commit/57a0530434b06c38a7862537b36ab61f46cdf183))
+* fix clean cache deleting config ([1edf493](https://github.com/windmill-labs/windmill/commit/1edf493e6d76cc4abe9d2e5c8846946a3da6d730))
+
+## [1.297.1](https://github.com/windmill-labs/windmill/compare/v1.297.0...v1.297.1) (2024-03-26)
+
+
+### Bug Fixes
+
+* fix approval steps handling of default args ([3388ab4](https://github.com/windmill-labs/windmill/commit/3388ab41ddaaf1a30b27292db11e650257990aa1))
+
+## [1.297.0](https://github.com/windmill-labs/windmill/compare/v1.296.1...v1.297.0) (2024-03-26)
+
+
+### Features
+
+* add visible to runner only default value ([#3472](https://github.com/windmill-labs/windmill/issues/3472)) ([14a86bf](https://github.com/windmill-labs/windmill/commit/14a86bf59c93d7e54d175a6e1e2b77804681fa3d))
+
+## [1.296.1](https://github.com/windmill-labs/windmill/compare/v1.296.0...v1.296.1) (2024-03-26)
+
+
+### Bug Fixes
+
+* **backend:** update chrono dependency ([703c118](https://github.com/windmill-labs/windmill/commit/703c11891a18ced5a8da4365426cf80614fc230c))
+
 ## [1.296.0](https://github.com/windmill-labs/windmill/compare/v1.295.4...v1.296.0) (2024-03-26)
 
 

Dockerfile

@@ -1,30 +1,6 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
 ARG RUST_IMAGE=rust:1.76-slim-bookworm
-ARG PYTHON_IMAGE=python:3.11.4-slim-bookworm
-
-FROM ${DEBIAN_IMAGE} as nsjail
-
-WORKDIR /nsjail
-
-ARG nsjail=""
-
-RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
-    && apt-get install -y \
-    bison=2:3.8.* \
-    flex=2.6.* \
-    g++=4:12.2.* \
-    gcc=4:12.2.* \
-    git=1:2.39.* \
-    libprotobuf-dev=3.21.* \
-    libnl-route-3-dev=3.7.* \
-    make=4.3-4.1 \
-    pkg-config=1.8.* \
-    protobuf-compiler=3.21.*; fi
-
-
-RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
-    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
-RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
+ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
 
@@ -100,8 +76,8 @@ SHELL ["/bin/bash", "-c"]
 RUN apt update -y
 RUN apt install -y unzip curl
 
-RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.41.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
-RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.41.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.42.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.42.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true
 
 
 RUN unzip deno.zip && rm deno.zip
@@ -111,8 +87,8 @@ FROM ${PYTHON_IMAGE}
 ARG TARGETPLATFORM
 ARG POWERSHELL_VERSION=7.3.5
 ARG POWERSHELL_DEB_VERSION=7.3.5-1
-ARG KUBECTL_VERSION=1.27.2
-ARG HELM_VERSION=3.12.0
+ARG KUBECTL_VERSION=1.28.7
+ARG HELM_VERSION=3.14.3
 ARG APP=/usr/src/app
 ARG WITH_POWERSHELL=true
 ARG WITH_KUBECTL=true
@@ -120,9 +96,10 @@ ARG WITH_HELM=true
 
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential unixodbc xmlsec1 \
+    && apt-get install -y ca-certificates wget curl git jq unzip build-essential unixodbc xmlsec1  software-properties-common \
     && rm -rf /var/lib/apt/lists/*
 
+
 RUN if [ "$WITH_POWERSHELL" = "true" ]; then \
     if [ "$TARGETPLATFORM" = "linux/amd64" ]; then apt-get update -y && apt install libicu-dev -y && wget -O 'pwsh.deb' "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/powershell_${POWERSHELL_DEB_VERSION}.deb_amd64.deb" && \
     dpkg --install 'pwsh.deb' && \
@@ -150,22 +127,6 @@ RUN if [ "$WITH_KUBECTL" = "true" ]; then \
     install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl; \
     else echo 'Building the image without kubectl'; fi
 
-RUN set -eux; \
-    arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
-    case "$arch" in \
-    'amd64') \
-    zip='awscli-exe-linux-x86_64.zip'; \
-    ;; \
-    'arm64') \
-    zip='awscli-exe-linux-aarch64.zip'; \
-    ;; \
-    *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
-    esac; \
-    apt-get update && apt install unzip && curl "https://awscli.amazonaws.com/$zip" -o "awscliv2.zip" && \
-    unzip awscliv2.zip && \
-    ./aws/install && rm awscliv2.zip
-
-
 
 RUN set -eux; \
     arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
@@ -186,14 +147,11 @@ RUN set -eux; \
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
-ARG nsjail=""
-
-RUN if [ "$nsjail" = "true" ]; then apt-get -y update \
-    && apt-get install -y \
-    curl nodejs npm; fi
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - 
+RUN apt-get -y update && apt-get install -y curl nodejs awscli
 
 # go build is slower the first time it is ran, so we prewarm it in the build
-RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm && printf "package foo\nimport (\"fmt\"\nwmill \"github.com/windmill-labs/windmill-go-client\")\nfunc main() { v := wmill.GetStatePath()\n fmt.Println(v) }" > warm.go && go mod tidy && go build -x && rm -rf /tmp/gobuildwarm
+RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm && printf "package foo\nimport (\"fmt\")\nfunc main() { fmt.Println(42) }" > warm.go && go mod tidy && go build -x && rm -rf /tmp/gobuildwarm
 
 ENV TZ=Etc/UTC
 
@@ -205,9 +163,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=downloader --chmod=755 /deno /usr/bin/deno
 
-COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
-COPY --from=oven/bun:1.0.29 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.0 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

backend/.sqlx/query-020d33ed5d47350b456783fd548422ea8dcf2d786d0e9fa849754db82c9fa378.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -65,10 +65,11 @@
         "Bool",
         "Bool",
         "Int4",
-        "Varchar"
+        "Varchar",
+        "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "260feb784bb0b223bd9276d6a82bd26be90efb17a4323b9673e93ff88513942a"
+  "hash": "020d33ed5d47350b456783fd548422ea8dcf2d786d0e9fa849754db82c9fa378"
 }

backend/.sqlx/query-0c349ac832cbf055fce755c8e2081eacf2b836b0396ec48a6cc2405fb7aa76de.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET suspend = suspend - 1 WHERE id = $1 AND suspend > 0",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0c349ac832cbf055fce755c8e2081eacf2b836b0396ec48a6cc2405fb7aa76de"
+}

backend/.sqlx/query-1289e7278d2a289bfaa53f00e0b6dceb195df0fb43a8ac03bc8b35939fc941dd.json

@@ -27,11 +27,6 @@
         "ordinal": 4,
         "name": "premium",
         "type_info": "Bool"
-      },
-      {
-        "ordinal": 5,
-        "name": "is_overquota",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -45,7 +40,6 @@
       false,
       false,
       false,
-      false,
       false
     ]
   },

backend/.sqlx/query-12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "12a0fd7d8d99fb73b01bc24774fe9a8da57b5204bb6b1207aed47143c17a20bc"
+}

backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n                    VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n                    RETURNING usage.usage",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "usage",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31"
+}

backend/.sqlx/query-35e6af0b203e3e4fac9020b037a3c41af92537c0fd5683227767f6a1bd17339f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema, dependency_job, draft_only, tag, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8, $9, $10)",
+  "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema, dependency_job, draft_only, tag, dedicated_worker, visible_to_runner_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8, $9, $10, $11)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -14,10 +14,11 @@
         "Text",
         "Bool",
         "Varchar",
+        "Bool",
         "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "c3da506f47dfb42434debf6e005319147c7513193e853c38cfb7c071d61a3d2e"
+  "hash": "35e6af0b203e3e4fac9020b037a3c41af92537c0fd5683227767f6a1bd17339f"
 }

backend/.sqlx/query-43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO workspace_settings\n                 (workspace_id, slack_team_id, slack_name, slack_email)\n                 VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, slack_name = $3, slack_email = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "43b376a2eff086a32cd76e54361ce3631feee1565935d2a6ddbecc17950758d1"
+}

backend/.sqlx/query-489a62b5943a7a21ce487aa7b72a63dfc6300dd93bc29f5ec4cb1bfc471ad0bf.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO resource\n             (workspace_id, path, value, description, resource_type)\n             VALUES ($1, $2, $3, $4, $5) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Jsonb",
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "489a62b5943a7a21ce487aa7b72a63dfc6300dd93bc29f5ec4cb1bfc471ad0bf"
+}

backend/.sqlx/query-52c8b4350235bdaab4df79e517d5e42a61a4e1e209d120b2c8bb31ebb7ce1e56.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO variable\n            (workspace_id, path, value, is_secret, description, account, is_oauth)\n            VALUES ($1, $2, $3, $4, $5, $6, $7)\n            ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Bool",
-        "Varchar",
-        "Int4",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "52c8b4350235bdaab4df79e517d5e42a61a4e1e209d120b2c8bb31ebb7ce1e56"
-}

backend/.sqlx/query-55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO variable\n             (workspace_id, path, value, is_secret, description, account, is_oauth)\n             VALUES ($1, $2, $3, $4, $5, $6, $7)\n             ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Varchar",
+        "Int4",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "55a2f170823f1d1abce76287d8817a6cf34de92b9b4079c00b75423a9ff835b9"
+}

backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT * FROM workspace_settings WHERE slack_team_id = $1",
+  "query": "SELECT * FROM workspace_settings WHERE slack_team_id = $1 AND slack_command_script IS NOT NULL",
   "describe": {
     "columns": [
       {
@@ -144,5 +144,5 @@
       true
     ]
   },
-  "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"
+  "hash": "55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2"
 }

backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                    VALUES ($1, $2, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) \n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n                    RETURNING usage.usage",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n                    VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n                    RETURNING usage.usage",
   "describe": {
     "columns": [
       {
@@ -11,13 +11,12 @@
     ],
     "parameters": {
       "Left": [
-        "Varchar",
-        "Bool"
+        "Varchar"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "bd086c56ad3a58163619122641b93b941ef75880eeeab0756d40f68f87760d5b"
+  "hash": "621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e"
 }

backend/.sqlx/query-6e816fdba0d281d5147ddc5d4d438a3209b811324dba4ac661938e0523210682.json

@@ -1,28 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT premium, is_overquota FROM workspace WHERE id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "premium",
-        "type_info": "Bool"
-      },
-      {
-        "ordinal": 1,
-        "name": "is_overquota",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false
-    ]
-  },
-  "hash": "6e816fdba0d281d5147ddc5d4d438a3209b811324dba4ac661938e0523210682"
-}

backend/.sqlx/query-73ca6ea13b362b2569ea115d28e4e255cd5e9b990ffa89998ef24871d3a9717c.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings\n             SET slack_team_id = null, slack_name = null WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "73ca6ea13b362b2569ea115d28e4e255cd5e9b990ffa89998ef24871d3a9717c"
+}

backend/.sqlx/query-7d93eb90163516718c85f28f8f05093133c5cbc96414000ac364583114d0ce77.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT usage.usage + 1 FROM usage \n                        WHERE is_workspace IS FALSE AND\n                        month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n                        AND id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "7d93eb90163516718c85f28f8f05093133c5cbc96414000ac364583114d0ce77"
+}

backend/.sqlx/query-82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n    SELECT usage.usage FROM usage \n    WHERE is_workspace = true \n    AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n    AND id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "usage",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8"
+}

backend/.sqlx/query-8f0031533f1bf407bd5d8af4d364eaf00d4c38ee7ba75141b40fc9fcd2ffc0b8.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO workspace SELECT $1, $2, owner, deleted, premium, is_overquota FROM workspace WHERE id = $3",
+  "query": "INSERT INTO workspace SELECT $1, $2, owner, deleted, premium FROM workspace WHERE id = $3",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "6d89ac43c4ae3a17167a44c802df72bc506803b79fe93ab3b203fb0690b1bef1"
+  "hash": "8f0031533f1bf407bd5d8af4d364eaf00d4c38ee7ba75141b40fc9fcd2ffc0b8"
 }

backend/.sqlx/query-95ebdfaf0510b9cad861568cd25d479759d4ea3c3ff4e136aad13a3521525372.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO folder\n                (workspace_id, name, display_name, owners, extra_perms)\n                VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "VarcharArray",
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "95ebdfaf0510b9cad861568cd25d479759d4ea3c3ff4e136aad13a3521525372"
-}

backend/.sqlx/query-a3ccf362b4f6df400b3c7a084795dbf541eb14c5c374656ffb96da7283a2a6f1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT concurrency_key FROM script WHERE hash = $1",
+  "query": "SELECT concurrency_key FROM script WHERE hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -11,12 +11,13 @@
     ],
     "parameters": {
       "Left": [
-        "Int8"
+        "Int8",
+        "Text"
       ]
     },
     "nullable": [
       true
     ]
   },
-  "hash": "2719f910142b32476a16025bb9836b0cab019ba0a436b330ea3a53fba4725f73"
+  "hash": "a3ccf362b4f6df400b3c7a084795dbf541eb14c5c374656ffb96da7283a2a6f1"
 }

backend/.sqlx/query-a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT usage.usage + 1 FROM usage \n                        WHERE is_workspace IS TRUE AND\n                        month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n                        AND id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a405e637f5f3b3203de6d65dfcb0ba1be406ee5167f7b8aa90213ef52c97441f"
+}

backend/.sqlx/query-a4ae245dcf7e4b930cd45701db0b7c45f2a5797e8b6724bade5b964d4334c098.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET suspend = suspend - 1 WHERE parent_job = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "a4ae245dcf7e4b930cd45701db0b7c45f2a5797e8b6724bade5b964d4334c098"
-}

backend/.sqlx/query-a875cb56485b812e9d4739afd0915067f7e5abe0ca0adf264b792fccf21e005b.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value->>'concurrency_key' FROM flow WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a875cb56485b812e9d4739afd0915067f7e5abe0ca0adf264b792fccf21e005b"
+}

backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4"
+}

backend/.sqlx/query-b8d97d300ffe6fae99f2396ae07cef03903752d17ce3fdadca47d86da75139aa.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id FROM queue WHERE parent_job = $1 AND suspend > 0 ORDER by suspend",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "b8d97d300ffe6fae99f2396ae07cef03903752d17ce3fdadca47d86da75139aa"
+}

backend/.sqlx/query-bf1d8e043338867e1da1ed236ff6c85a566d5fd58d4b0d5c3a10454513811ba3.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings\n            SET slack_team_id = null, slack_name = null WHERE workspace_id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "bf1d8e043338867e1da1ed236ff6c85a566d5fd58d4b0d5c3a10454513811ba3"
-}

backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8"
+}

backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO completed_job AS cj\n                   ( workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , started_at\n                   , duration_ms\n                   , success\n                   , script_hash\n                   , script_path\n                   , args\n                   , result\n                   , raw_code\n                   , raw_lock\n                   , canceled\n                   , canceled_by\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , is_skipped\n                   , language\n                   , email\n                   , visible_to_owner\n                   , mem_peak\n                   , tag\n                   , priority\n                )\n            VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($25, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30)\n         ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
+  "query": "INSERT INTO completed_job AS cj\n                   ( workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , started_at\n                   , duration_ms\n                   , success\n                   , script_hash\n                   , script_path\n                   , args\n                   , result\n                   , raw_code\n                   , raw_lock\n                   , canceled\n                   , canceled_by\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , is_skipped\n                   , language\n                   , email\n                   , visible_to_owner\n                   , mem_peak\n                   , tag\n                   , priority\n                )\n            VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000, $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)\n         ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
   "describe": {
     "columns": [
       {
@@ -79,7 +79,6 @@
             }
           }
         },
-        "Numeric",
         "Varchar",
         "Bool",
         "Int4",
@@ -91,5 +90,5 @@
       false
     ]
   },
-  "hash": "2ade671449393541fa565088b21268dad137314d250f7ded502defb9a6de0b2f"
+  "hash": "d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1"
 }

backend/.sqlx/query-d918e1d3a1ccc36ca8b4b25aeeec1cb55aadaf1e6609a1aad09b0515c93a667a.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n        SELECT usage.usage + 1 FROM usage \n        WHERE is_workspace = false AND\n     month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n     AND id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "?column?",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "d918e1d3a1ccc36ca8b4b25aeeec1cb55aadaf1e6609a1aad09b0515c93a667a"
-}

backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c"
+}

backend/.sqlx/query-e7418515b88d14d1fd79078342963e11422eed9a60e0472c68971f4e8f8735d9.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n                VALUES ($1, $2, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) \n                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Bool",
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "e7418515b88d14d1fd79078342963e11422eed9a60e0472c68971f4e8f8735d9"
-}

backend/.sqlx/query-e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(id) FROM queue WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e7a1c2b5d79e72f557181782419a9d8d1a502796842f185374d2d0f69043086b"
+}

backend/.sqlx/query-ea8ebb8d972fe99c960b5a69f794ee2b57bfb1914bf370c5b10313e45fa9b65f.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO resource\n            (workspace_id, path, value, description, resource_type)\n            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Jsonb",
-        "Text",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "ea8ebb8d972fe99c960b5a69f794ee2b57bfb1914bf370c5b10313e45fa9b65f"
-}

backend/.sqlx/query-eaa6e9dc4c0d3d6cc8152515019befa880bb3b69ff340d337edd14b65e74e2a3.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO workspace_settings\n                (workspace_id, slack_team_id, slack_name, slack_email)\n                VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, slack_name = $3, slack_email = $4",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "eaa6e9dc4c0d3d6cc8152515019befa880bb3b69ff340d337edd14b65e74e2a3"
-}

backend/.sqlx/query-f2aee3ae39c90e40dd1835befc339e5381cc9104933cf8e90d840a9bf638ff52.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder\n                 (workspace_id, name, display_name, owners, extra_perms)\n                 VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "VarcharArray",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f2aee3ae39c90e40dd1835befc339e5381cc9104933cf8e90d840a9bf638ff52"
+}

backend/.sqlx/query-f516ca558816c2cbab3c8ae865ef8f764aa4686e7df9c53d30d49a3dbbf36af4.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT SUM(duration_ms) as duration FROM completed_job WHERE id = ANY($1)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "duration",
-        "type_info": "Numeric"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "UuidArray"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "f516ca558816c2cbab3c8ae865ef8f764aa4686e7df9c53d30d49a3dbbf36af4"
-}

backend/.sqlx/query-f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                SELECT EXISTS (SELECT 1 \n                FROM workspace_settings \n                WHERE workspace_id <> $1 \n                    AND slack_command_script IS NOT NULL\n                    AND slack_team_id IS NOT NULL \n                    AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1))\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4"
+}

backend/.sqlx/query-f6fd65fbe36502923ab4ccf1a22f748cb854e23049d0cf73a42229eccc88c4e6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL, tag = $9, dedicated_worker = $10\n        WHERE path = $7 AND workspace_id = $8",
+  "query": "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL, tag = $9, dedicated_worker = $10, visible_to_runner_only = $11\n        WHERE path = $7 AND workspace_id = $8",
   "describe": {
     "columns": [],
     "parameters": {
@@ -14,10 +14,11 @@
         "Text",
         "Text",
         "Varchar",
+        "Bool",
         "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "de06474de29c6c2cadb99787f5d627678a5285011d7b1a516d4e240a5810ddba"
+  "hash": "f6fd65fbe36502923ab4ccf1a22f748cb854e23049d0cf73a42229eccc88c4e6"
 }

backend/.sqlx/query-f7f2598ac824e9b5719b96b1d1873ad728f7f786c7f447df6cac3e65d5a53bf1.json

@@ -27,11 +27,6 @@
         "ordinal": 4,
         "name": "premium",
         "type_info": "Bool"
-      },
-      {
-        "ordinal": 5,
-        "name": "is_overquota",
-        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -44,7 +39,6 @@
       false,
       false,
       false,
-      false,
       false
     ]
   },

backend/.sqlx/query-fa59674af1d1a4ceb696fc883005ef114772f7d2ee0f60cb1358cdb7f0b5cd0c.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n             SELECT EXISTS (SELECT 1 \n             FROM workspace_settings \n             WHERE workspace_id <> $1 \n                 AND slack_command_script IS NOT NULL\n                 AND slack_team_id = $2\n                 AND (SELECT slack_command_script IS NOT NULL FROM workspace_settings WHERE workspace_id = $1))\n         ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "fa59674af1d1a4ceb696fc883005ef114772f7d2ee0f60cb1358cdb7f0b5cd0c"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.296.0"
+version = "1.305.3"
 authors.workspace = true
 edition.workspace = true
 
@@ -20,10 +20,11 @@ members = [
     "./parsers/windmill-parser-bash",
     "./parsers/windmill-parser-py",
     "./parsers/windmill-parser-py-imports",
+    "./parsers/windmill-sql-datatype-parser-wasm",
 ]
 
 [workspace.package]
-version = "1.296.0"
+version = "1.305.3"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -119,7 +120,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
-chrono = { version = "^0", features = ["serde"] }
+chrono = { version = "0.4.35", features = ["serde"] }
 chrono-tz = "^0"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
@@ -143,7 +144,6 @@ urlencoding = "^2"
 url = "^2"
 async-oauth2 = "^0"
 reqwest = { version = "^0.12", features = ["json", "stream"] }
-reqwest_11 = { package = "reqwest", version = "0.11" }
 time = "0.3.16"
 serde_urlencoded = "^0"
 tokio-tar = "^0"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-721b9298e35b831b390b7311b791a5b526dee923
+66d9cbb158ab9a5869a45ba253bf57f2cbb5ecb6

backend/migrations/20240326122255_add_visible_to_runner_only.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240326122255_add_visible_to_runner_only.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN visible_to_runner_only BOOLEAN;
+ALTER TABLE flow ADD COLUMN visible_to_runner_only BOOLEAN;

backend/migrations/20240328145425_remove_overquota.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240328145425_remove_overquota.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+alter table workspace drop column is_overquota;

backend/migrations/20240331125020_webhooks_audits.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240331125020_webhooks_audits.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+CREATE POLICY "webhook" ON audit FOR INSERT
+TO windmill_user
+WITH CHECK (((username)::text ~~ 'webhook-%'::text))

backend/migrations/20240403083110_remove_team_id_constraint.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240403083110_remove_team_id_constraint.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE public.workspace_settings DROP CONSTRAINT workspace_settings_slack_team_id_key

backend/migrations/20240408134243_longer_approvers.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240408134243_longer_approvers.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE resume_job ALTER COLUMN approver TYPE VARCHAR(1000);

backend/migrations/20240409181630_add_concurrency_key_to_flow.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE flow DROP COLUMN concurrency_key;

backend/migrations/20240409181630_add_concurrency_key_to_flow.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE flow ADD COLUMN concurrency_key VARCHAR(255);

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -5,7 +5,7 @@ use regex::Regex;
 use serde_json::json;
 
 use std::collections::HashMap;
-use windmill_parser::{Arg, MainArgSignature, Typ};
+pub use windmill_parser::{Arg, MainArgSignature, Typ};
 
 pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_mysql_file(&code)?;

backend/parsers/windmill-sql-datatype-parser-wasm/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "windmill-sql-datatype-parser-wasm"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+
+
+[lib]
+crate-type = ["cdylib"]
+name = "windmill_sql_datatype_parser_wasm"
+path = "./src/lib.rs"
+
+[dev-dependencies]
+wasm-bindgen-test.workspace = true
+
+[dependencies]
+windmill-parser.workspace = true
+windmill-parser-sql.workspace = true
+wasm-bindgen.workspace = true
+serde = { version = "1.0", features = ["derive"] }

backend/parsers/windmill-sql-datatype-parser-wasm/build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -eou pipefail
+
+deno task wasmbuild --out ../../../cli/wasm/

backend/parsers/windmill-sql-datatype-parser-wasm/package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "windmill-parser-wasm",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {}
+}

backend/parsers/windmill-sql-datatype-parser-wasm/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "windmill-sql-datatype-parser-wasm",
+  "version": "0.1.0"
+}

backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs

@@ -0,0 +1,40 @@
+use wasm_bindgen::prelude::*;
+use windmill_parser_sql::Typ;
+
+fn to_str(typ: Typ) -> String {
+    match typ {
+        Typ::Str(_) => "str".to_string(),
+        Typ::Int => "int".to_string(),
+        Typ::Float => "float".to_string(),
+        Typ::Bool => "bool".to_string(),
+        Typ::List(t) => format!("list-{}", to_str(*t)),
+        Typ::Bytes => "bytes".to_string(),
+        Typ::Datetime => "datetime".to_string(),
+        _ => "unknown".to_string(),
+    }
+}
+
+#[wasm_bindgen]
+pub fn parse_sql(typ: &str) -> String {
+    to_str(windmill_parser_sql::parse_pg_typ(typ))
+}
+
+#[wasm_bindgen]
+pub fn parse_mysql(typ: &str) -> String {
+    to_str(windmill_parser_sql::parse_mysql_typ(typ))
+}
+
+#[wasm_bindgen]
+pub fn parse_bigquery(typ: &str) -> String {
+    to_str(windmill_parser_sql::parse_bigquery_typ(typ))
+}
+
+#[wasm_bindgen]
+pub fn parse_snowflake(typ: &str) -> String {
+    to_str(windmill_parser_sql::parse_snowflake_typ(typ))
+}
+
+#[wasm_bindgen]
+pub fn parse_mssql(typ: &str) -> String {
+    to_str(windmill_parser_sql::parse_mssql_typ(typ))
+}

backend/src/main.rs

@@ -19,8 +19,8 @@ use windmill_api::HTTP_CLIENT;
 use windmill_common::{
     global_settings::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CUSTOM_TAGS_SETTING,
-        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS,
-        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
+        EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING,
         JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
         NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
         REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
@@ -47,10 +47,10 @@ use windmill_worker::{
 use crate::monitor::{
     initial_load, load_keep_job_dir, load_require_preexisting_user, load_tag_per_workspace_enabled,
     monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting,
-    reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key,
-    reload_npm_config_registry_setting, reload_pip_index_url_setting,
-    reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
-    reload_worker_config,
+    reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
+    reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting,
+    reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
+    reload_server_config, reload_worker_config,
 };
 
 #[cfg(feature = "parquet")]
@@ -498,7 +498,11 @@ Windmill Community Edition {GIT_VERSION}
                                                         tracing::error!(error = %e, "Could not send killpill to server");
                                                     }
                                                 },
-                                                DISABLE_STATS_SETTING => {},
+                                                HUB_BASE_URL_SETTING => {
+                                                    if let Err(e) = reload_hub_base_url_setting(&db, server_mode).await {
+                                                        tracing::error!(error = %e, "Could not reload hub base url setting");
+                                                    }
+                                                },
                                                 a @_ => {
                                                     tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
                                                 }

backend/src/monitor.rs

@@ -14,6 +14,9 @@ use tokio::{
     join,
     sync::{mpsc, RwLock},
 };
+
+#[cfg(feature = "embedding")]
+use windmill_api::embeddings::update_embeddings_db;
 use windmill_api::{
     oauth2_ee::{build_oauth_clients, OAuthClient},
     DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
@@ -24,8 +27,8 @@ use windmill_common::{
     global_settings::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
-        NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
+        HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
         REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
         RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
     },
@@ -37,7 +40,7 @@ use windmill_common::{
         load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG,
         WORKER_CONFIG,
     },
-    BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
+    BASE_URL, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
 };
 use windmill_queue::cancel_job;
 use windmill_worker::{
@@ -136,6 +139,10 @@ pub async fn initial_load(
         tracing::error!("Error reloading base url: {:?}", e)
     }
 
+    if let Err(e) = reload_hub_base_url_setting(db, server_mode).await {
+        tracing::error!("Error reloading hub base url: {:?}", e)
+    }
+
     #[cfg(feature = "parquet")]
     if !_is_agent {
         reload_s3_cache_setting(&db).await;
@@ -920,7 +927,7 @@ async fn handle_zombie_flows(
         r#"
         SELECT *
         FROM queue
-        WHERE running = true AND suspend = 0 AND scheduled_for <= now() AND (job_kind = 'flow' OR job_kind = 'flowpreview')
+        WHERE running = true AND suspend = 0 AND suspend_until IS null AND scheduled_for <= now() AND (job_kind = 'flow' OR job_kind = 'flowpreview')
             AND last_ping IS NOT NULL AND last_ping < NOW() - ($1 || ' seconds')::interval 
         "#,
     ).bind(FLOW_ZOMBIE_TRANSITION_TIMEOUT.as_str())
@@ -1025,3 +1032,45 @@ async fn cancel_zombie_flow_job(
     ntx.commit().await?;
     Ok(())
 }
+
+pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> {
+    let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?;
+
+    let base_url = if let Some(q) = hub_base_url {
+        if let Ok(v) = serde_json::from_value::<String>(q.clone()) {
+            if v != "" {
+                v
+            } else {
+                DEFAULT_HUB_BASE_URL.to_string()
+            }
+        } else {
+            tracing::error!(
+                "Could not parse hub_base_url setting as a string, found: {:#?}",
+                &q
+            );
+            DEFAULT_HUB_BASE_URL.to_string()
+        }
+    } else {
+        DEFAULT_HUB_BASE_URL.to_string()
+    };
+
+    let mut l = HUB_BASE_URL.write().await;
+    if server_mode {
+        #[cfg(feature = "embedding")]
+        if *l != base_url {
+            let disable_embedding = std::env::var("DISABLE_EMBEDDING")
+                .ok()
+                .map(|x| x.parse::<bool>().unwrap_or(false))
+                .unwrap_or(false);
+            if !disable_embedding {
+                let db_clone = db.clone();
+                tokio::spawn(async move {
+                    update_embeddings_db(&db_clone).await;
+                });
+            }
+        }
+    }
+    *l = base_url;
+
+    Ok(())
+}

backend/tests/worker.rs

@@ -1079,7 +1079,8 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
-                    delete_after_use: None
+                    delete_after_use: None,
+                    continue_on_error: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1116,6 +1117,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                             timeout: None,
                             priority: None,
                             delete_after_use: None,
+                            continue_on_error: None,
                         }],
                     },
                     stop_after_if: Default::default(),
@@ -1128,6 +1130,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
             ],
             same_worker: false,
@@ -1229,6 +1232,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1276,6 +1280,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 timeout: None,
                                 priority: None,
                                 delete_after_use: None,
+                                continue_on_error: None,
                             },
                             FlowModule {
                                 id: "e".to_string(),
@@ -1308,7 +1313,9 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 mock: None,
                                 timeout: None,
                                 priority: None,
-                                delete_after_use: None
+                                delete_after_use: None,
+                                continue_on_error: None,
+
                             },
                         ],
                     },
@@ -1321,7 +1328,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
-                    delete_after_use: None
+                    delete_after_use: None,
+                    continue_on_error: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1362,6 +1370,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
             ],
             same_worker: true,
@@ -2582,6 +2591,7 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
                     priority: None,
                     dedicated_worker: None,
                     timeout: None,
+                    visible_to_runner_only: None,
                 },
                 draft_only: None,
                 deployment_message: None,
@@ -3153,6 +3163,7 @@ async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: Stri
             restart_unless_cancelled: None,
             deployment_message: None,
             concurrency_key: None,
+            visible_to_runner_only: None,
         },
     ).await.unwrap();
 

backend/windmill-api/Cargo.toml

@@ -39,7 +39,6 @@ tower-http.workspace = true
 hyper.workspace = true
 itertools.workspace = true
 reqwest.workspace = true
-reqwest_11.workspace = true
 serde.workspace = true
 sqlx.workspace = true
 async-oauth2.workspace = true

backend/windmill-api/build_openapi.sh

@@ -5,5 +5,7 @@ npx @redocly/openapi-cli@latest bundle openapi.yaml > openapi-bundled.yaml
 
 
 npx @redocly/openapi-cli@latest bundle openapi-bundled.yaml --ext yaml -d > openapi-deref.yaml
+npx @redocly/openapi-cli@latest bundle openapi-bundled.yaml --ext json > openapi-deref.json
+
 
 rm openapi-bundled.yaml

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.296.0
+  version: 1.305.3
   title: Windmill API
 
   contact:
@@ -718,7 +718,6 @@ paths:
               schema:
                 type: string
 
-
     # pub use_ssl: Option<bool>,
     # #[serde(rename = "accountName")]
     # pub account_name: String,
@@ -1229,7 +1228,6 @@ paths:
               schema:
                 type: string
 
-
   /w/{workspace}/workspaces/change_workspace_id:
     post:
       summary: change workspace id
@@ -1957,6 +1955,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/LargeFileStorage"
 
+  /w/{workspace}/workspaces/usage:
+    get:
+      summary: get usage
+      operationId: getWorkspaceUsage
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: usage
+          content:
+            text/plain:
+              schema:
+                type: number
+
   /w/{workspace}/users/list:
     get:
       summary: list users
@@ -3876,6 +3890,7 @@ paths:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
+        - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
@@ -3939,6 +3954,7 @@ paths:
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
+        - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
@@ -3968,6 +3984,7 @@ paths:
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
+        - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
@@ -5025,6 +5042,7 @@ paths:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
+        - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
@@ -5216,6 +5234,8 @@ paths:
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
         - $ref: "#/components/parameters/Tag"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
         - name: all_workspaces
           description: get jobs from all workspaces (only valid if request come from the `admins` workspace)
           in: query
@@ -5318,6 +5338,8 @@ paths:
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
         - $ref: "#/components/parameters/Tag"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -5367,6 +5389,8 @@ paths:
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/Tag"
         - $ref: "#/components/parameters/ResultFilter"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -6176,6 +6200,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/Page"
         - $ref: "#/components/parameters/PerPage"
+        - $ref: "#/components/parameters/ArgsFilter"
         - name: path
           description: filter by path
           in: query
@@ -7995,6 +8020,12 @@ components:
       in: query
       schema:
         type: string
+    CacheTtl:
+      name: cache_ttl
+      description: Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl
+      in: query
+      schema:
+        type: string
     NewJobId:
       name: job_id
       description:
@@ -8303,6 +8334,8 @@ components:
           type: integer
         delete_after_use:
           type: boolean
+        visible_to_runner_only:
+          type: boolean
       required:
         - hash
         - path
@@ -8389,6 +8422,8 @@ components:
           type: string
         concurrency_key:
           type: string
+        visible_to_runner_only:
+          type: boolean
       required:
         - path
         - summary
@@ -9717,6 +9752,8 @@ components:
           type: boolean
         timeout:
           type: number
+        visible_to_runner_only:
+          type: boolean
       required:
         - path
         - edited_by
@@ -9741,6 +9778,8 @@ components:
               type: boolean
             timeout:
               type: number
+            visible_to_runner_only:
+              type: boolean
           required:
             - path
 
@@ -10097,6 +10136,8 @@ components:
           type: string
         use_individual_branch:
           type: boolean
+        group_by_folder:
+          type: boolean
         exclude_types_override:
           type: array
           items:

backend/windmill-api/src/apps.rs

@@ -39,6 +39,7 @@ use windmill_common::{
         http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
     },
     variables::build_crypt,
+    HUB_BASE_URL,
 };
 
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -633,16 +634,12 @@ async fn create_app(
 async fn list_hub_apps(Extension(db): Extension<DB>) -> impl IntoResponse {
     let (status_code, headers, body) = query_elems_from_hub(
         &HTTP_CLIENT,
-        "https://hub.windmill.dev/searchUiData?approved=true",
+        &format!("{}/searchUiData?approved=true", *HUB_BASE_URL.read().await),
         None,
         &db,
     )
     .await?;
-    Ok::<_, Error>((
-        status_code,
-        headers,
-        body
-    ))
+    Ok::<_, Error>((status_code, headers, body))
 }
 
 pub async fn get_hub_app_by_id(
@@ -651,7 +648,7 @@ pub async fn get_hub_app_by_id(
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
         &HTTP_CLIENT,
-        &format!("https://hub.windmill.dev/apps/{id}/json"),
+        &format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id),
         false,
         None,
         &db,

backend/windmill-api/src/db.rs

@@ -166,6 +166,7 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
             tracing::error!("Database had been applied more migrations than this container. 
             This usually mean than another container on a more recent version migrated the database and this one is on an earlier version.
             Please update the container to latest. Not critical, but may cause issues if migration introduced a breaking change. Version missing: {e}");
+            custom_migrator.unlock().await?;
             Ok(())
         }
         Err(err) => Err(err),

backend/windmill-api/src/embeddings.rs

@@ -1,21 +1,22 @@
 #[cfg(feature = "embedding")]
+use anyhow::{anyhow, Error, Result};
+#[cfg(feature = "embedding")]
 use std::{collections::HashMap, path::PathBuf, sync::Arc};
 #[cfg(feature = "embedding")]
-use anyhow::{anyhow, Error, Result};
+use windmill_common::DEFAULT_HUB_BASE_URL;
+#[cfg(feature = "embedding")]
+use windmill_common::HUB_BASE_URL;
 
 use axum::Router;
 
 #[cfg(feature = "embedding")]
 use axum::{
     extract::{Path, Query},
-    Json, 
+    Json,
 };
 
-
 #[cfg(feature = "embedding")]
-use axum::{
-    routing::get, Extension
-};
+use axum::routing::get;
 #[cfg(feature = "embedding")]
 use candle_core::{Device, Tensor};
 #[cfg(feature = "embedding")]
@@ -45,9 +46,14 @@ use windmill_common::utils::http_get_from_hub;
 use windmill_common::error::JsonResult;
 
 #[cfg(feature = "embedding")]
-
 use crate::{resources::ResourceType, HTTP_CLIENT};
 
+#[cfg(feature = "embedding")]
+lazy_static::lazy_static! {
+    pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
+    pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
+}
+
 #[cfg(feature = "embedding")]
 #[derive(Deserialize)]
 struct HubScriptsQuery {
@@ -71,9 +77,8 @@ pub struct HubScriptResult {
 #[cfg(feature = "embedding")]
 async fn query_hub_scripts(
     Query(query): Query<HubScriptsQuery>,
-    Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
 ) -> JsonResult<Vec<HubScriptResult>> {
-    let embeddings_db = embeddings_db.read().await;
+    let embeddings_db = EMBEDDINGS_DB.read().await;
 
     if let Some(embeddings_db) = embeddings_db.as_ref() {
         let results = embeddings_db
@@ -88,7 +93,6 @@ async fn query_hub_scripts(
     }
 }
 
-
 #[cfg(feature = "embedding")]
 #[derive(Deserialize)]
 struct ResourceTypesQuery {
@@ -106,9 +110,8 @@ pub struct ResourceTypeResult {
 async fn query_resource_types(
     Query(query): Query<ResourceTypesQuery>,
     Path(w_id): Path<String>,
-    Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
 ) -> JsonResult<Vec<ResourceTypeResult>> {
-    let embeddings_db = embeddings_db.read().await;
+    let embeddings_db = EMBEDDINGS_DB.read().await;
 
     if let Some(embeddings_db) = embeddings_db.as_ref() {
         let results = embeddings_db
@@ -123,7 +126,6 @@ async fn query_resource_types(
     }
 }
 
-
 #[cfg(feature = "embedding")]
 #[derive(Deserialize, Debug, Clone)]
 struct HubScript {
@@ -292,24 +294,41 @@ impl EmbeddingsDb {
         self.db
             .create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
 
-        let response = HTTP_CLIENT
-            .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
-            .send()
-            .await;
-        let response =
-            if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
-                tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
+        let hub_base_url = HUB_BASE_URL.read().await.clone();
+
+        let response = match hub_base_url.as_str() {
+            DEFAULT_HUB_BASE_URL => {
+                let response = HTTP_CLIENT
+                    .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
+                    .send()
+                    .await;
+
+                if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
+                    tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
+                    http_get_from_hub(
+                        &HTTP_CLIENT,
+                        &format!("{}/scripts/embeddings", hub_base_url),
+                        false,
+                        None,
+                        pg_db,
+                    )
+                    .await?
+                } else {
+                    response.unwrap()
+                }
+            }
+            _ => {
                 http_get_from_hub(
                     &HTTP_CLIENT,
-                    "https://hub.windmill.dev/scripts/embeddings",
+                    &format!("{}/scripts/embeddings", hub_base_url),
                     false,
                     None,
                     pg_db,
                 )
                 .await?
-            } else {
-                response.unwrap()
-            };
+            }
+        };
+
         if response.error_for_status_ref().is_err() {
             return Err(anyhow!(
                 "Failed to get scripts embeddings from hub with error code: {}",
@@ -338,25 +357,40 @@ impl EmbeddingsDb {
             self.db.insert_into_collection("scripts", embedding)?;
         }
 
-        let response = HTTP_CLIENT
-            .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
-            .send()
-            .await;
-        let response = if response.is_err()
-            || response.as_ref().unwrap().error_for_status_ref().is_err()
-        {
-            tracing::warn!("Failed to get resource types embeddings from bucket, trying hub...");
-            http_get_from_hub(
-                &HTTP_CLIENT,
-                "https://hub.windmill.dev/resource_types/embeddings",
-                false,
-                None,
-                pg_db,
-            )
-            .await?
-        } else {
-            response.unwrap()
+        let response = match hub_base_url.as_str() {
+            DEFAULT_HUB_BASE_URL => {
+                let response = HTTP_CLIENT
+                    .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
+                    .send()
+                    .await;
+                if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
+                    tracing::warn!(
+                        "Failed to get resource types embeddings from bucket, trying hub..."
+                    );
+                    http_get_from_hub(
+                        &HTTP_CLIENT,
+                        &format!("{}/resource_types/embeddings", hub_base_url),
+                        false,
+                        None,
+                        pg_db,
+                    )
+                    .await?
+                } else {
+                    response.unwrap()
+                }
+            }
+            _ => {
+                http_get_from_hub(
+                    &HTTP_CLIENT,
+                    &format!("{}/resource_types/embeddings", hub_base_url),
+                    false,
+                    None,
+                    pg_db,
+                )
+                .await?
+            }
         };
+
         if response.error_for_status_ref().is_err() {
             return Err(anyhow!(
                 "Failed to get resource types embeddings from hub with error code: {}",
@@ -554,9 +588,7 @@ fn normalize_l2(v: &Tensor) -> Result<Tensor> {
 }
 
 #[cfg(feature = "embedding")]
-pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb>>> {
-    let embeddings_db: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
-
+pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
     let disable_embedding = std::env::var("DISABLE_EMBEDDING")
         .ok()
         .map(|x| x.parse::<bool>().unwrap_or(false))
@@ -564,24 +596,14 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb
 
     if !disable_embedding {
         let db_clone = db.clone();
-        let embeddings_clone: Arc<RwLock<Option<EmbeddingsDb>>> = embeddings_db.clone();
         tokio::spawn(async move {
             let model_instance = ModelInstance::new().await;
-
             if let Ok(model_instance) = model_instance {
-                let model_instance = Arc::new(model_instance);
+                let mut model_instance_lock = MODEL_INSTANCE.write().await;
+                *model_instance_lock = Some(Arc::new(model_instance));
+                drop(model_instance_lock);
                 loop {
-                    tracing::info!("Creating embeddings DB...");
-                    let new_embeddings_db =
-                        EmbeddingsDb::new(&db_clone, model_instance.clone()).await;
-                    if let Err(e) = new_embeddings_db.as_ref() {
-                        tracing::error!("Failed to create embeddings db: {}", e);
-                    } else {
-                        let mut embeddings_db = embeddings_clone.write().await;
-                        *embeddings_db = new_embeddings_db.ok();
-                        tracing::info!("Created embeddings DB");
-                    }
-
+                    update_embeddings_db(&db_clone).await;
                     tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await;
                 }
             } else {
@@ -592,39 +614,41 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb
             }
         });
     }
-
-    embeddings_db
 }
 
 #[cfg(feature = "embedding")]
-pub fn workspaced_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
-    if let Some(embeddings_db) = embeddings_db {
-        Router::new()
-            .route("/query_resource_types", get(query_resource_types))
-            .layer(Extension(embeddings_db))
+pub async fn update_embeddings_db(db: &Pool<Postgres>) -> () {
+    if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() {
+        tracing::info!("Creating embeddings DB...");
+        let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await;
+        if let Err(e) = new_embeddings_db.as_ref() {
+            tracing::error!("Failed to create embeddings db: {}", e);
+        } else {
+            let mut embeddings_db = EMBEDDINGS_DB.write().await;
+            *embeddings_db = new_embeddings_db.ok();
+            tracing::info!("Created embeddings DB");
+        }
     } else {
-        Router::new()
+        tracing::error!("Could not update embeddings DB, model instance not initialized");
     }
 }
 
 #[cfg(feature = "embedding")]
-pub fn global_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
-    if let Some(embeddings_db) = embeddings_db {
-        Router::new()
-            .route("/query_hub_scripts", get(query_hub_scripts))
-            .layer(Extension(embeddings_db))
-    } else {
-        Router::new()
-    }
+pub fn workspaced_service() -> Router {
+    Router::new().route("/query_resource_types", get(query_resource_types))
 }
 
-
-#[cfg(not(feature = "embedding"))]
-pub fn workspaced_service(_embeddings_db: Option<()>) -> Router {
-        Router::new()
+#[cfg(feature = "embedding")]
+pub fn global_service() -> Router {
+    Router::new().route("/query_hub_scripts", get(query_hub_scripts))
 }
 
 #[cfg(not(feature = "embedding"))]
-pub fn global_service(_embeddings_db: Option<()>) -> Router {
-        Router::new()
+pub fn workspaced_service() -> Router {
+    Router::new()
+}
+
+#[cfg(not(feature = "embedding"))]
+pub fn global_service() -> Router {
+    Router::new()
 }

backend/windmill-api/src/flows.rs

@@ -31,6 +31,7 @@ use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::audit_ee::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::utils::query_elems_from_hub;
+use windmill_common::HUB_BASE_URL;
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
@@ -167,7 +168,10 @@ async fn list_flows(
 async fn list_hub_flows(Extension(db): Extension<DB>) -> impl IntoResponse {
     let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
-        "https://hub.windmill.dev/searchFlowData?approved=true",
+        &format!(
+            "{}/searchFlowData?approved=true",
+            *HUB_BASE_URL.read().await
+        ),
         None,
         &db,
     )
@@ -199,7 +203,7 @@ pub async fn get_hub_flow_by_id(
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
         &HTTP_CLIENT,
-        &format!("https://hub.windmill.dev/flows/{id}/json"),
+        &format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id),
         false,
         None,
         &db,
@@ -319,7 +323,7 @@ async fn create_flow(
 
     sqlx::query!(
         "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema, dependency_job, draft_only, tag, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8, $9, $10)",
+         schema, dependency_job, draft_only, tag, dedicated_worker, visible_to_runner_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8, $9, $10, $11)",
         w_id,
         nf.path,
         nf.summary,
@@ -329,7 +333,8 @@ async fn create_flow(
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
         nf.draft_only,
         nf.tag,
-        nf.dedicated_worker
+        nf.dedicated_worker,
+        nf.visible_to_runner_only.unwrap_or(false),
     )
     .execute(&mut tx)
     .await?;
@@ -485,7 +490,7 @@ async fn update_flow(
     let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
     sqlx::query!(
         "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL, tag = $9, dedicated_worker = $10
+         edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL, tag = $9, dedicated_worker = $10, visible_to_runner_only = $11
         WHERE path = $7 AND workspace_id = $8",
         nf.path,
         nf.summary,
@@ -496,7 +501,8 @@ async fn update_flow(
         flow_path,
         w_id,
         nf.tag,
-        nf.dedicated_worker
+        nf.dedicated_worker,
+        nf.visible_to_runner_only.unwrap_or(false),
     )
     .execute(&mut tx)
     .await?;
@@ -669,6 +675,8 @@ pub struct FlowWDraft {
     pub ws_error_handler_muted: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub dedicated_worker: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
 }
 
 async fn get_flow_by_path_w_draft(
@@ -680,7 +688,7 @@ async fn get_flow_by_path_w_draft(
     let mut tx = user_db.begin(&authed).await?;
 
     let flow_o = sqlx::query_as::<_, FlowWDraft>(
-        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag
+        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, flow.ws_error_handler_muted, flow.dedicated_worker, draft.value as draft, flow.tag, flow.visible_to_runner_only
          FROM flow
         LEFT JOIN draft ON 
         flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' 
@@ -893,6 +901,7 @@ mod tests {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -919,6 +928,7 @@ mod tests {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -942,6 +952,7 @@ mod tests {
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 },
             ],
             failure_module: Some(FlowModule {
@@ -965,6 +976,7 @@ mod tests {
                 timeout: None,
                 priority: None,
                 delete_after_use: None,
+                continue_on_error: None,
             }),
             same_worker: false,
             concurrent_limit: None,
@@ -973,6 +985,7 @@ mod tests {
             cache_ttl: None,
             priority: None,
             early_return: None,
+            concurrency_key: None,
         };
         let expect = serde_json::json!({
           "modules": [

backend/windmill-api/src/integration.rs

@@ -1,8 +1,6 @@
 use crate::{db::DB, HTTP_CLIENT};
-use axum::{
-     extract::Query, response::IntoResponse, routing::get, Extension, Router,
-};
-use windmill_common::{error::Error, utils::query_elems_from_hub};
+use axum::{extract::Query, response::IntoResponse, routing::get, Extension, Router};
+use windmill_common::{error::Error, utils::query_elems_from_hub, HUB_BASE_URL};
 
 pub fn global_service() -> Router {
     Router::new().route("/hub/list", get(list_hub_integrations))
@@ -24,14 +22,10 @@ async fn list_hub_integrations(
 
     let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
-        "https://hub.windmill.dev/integrations/list",
+        &format!("{}/integrations/list", *HUB_BASE_URL.read().await),
         Some(query_params),
         &db,
     )
     .await?;
-    Ok::<_, Error>((
-        status_code,
-        headers,
-        response
-    ))
+    Ok::<_, Error>((status_code, headers, response))
 }

backend/windmill-api/src/jobs.rs

@@ -662,21 +662,29 @@ async fn get_logs_from_store(
 ) -> Option<error::Result<Body>> {
     if log_offset > 0 {
         if let Some(file_index) = log_file_index {
+            tracing::debug!("Getting logs from store: {file_index:?}");
             if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
+                tracing::debug!("object store client present, streaming from there");
+
                 let logs = logs.to_string();
                 let stream = async_stream::stream! {
-                    for file in file_index {
-                        let file = os.get(&object_store::path::Path::from(file)).await;
+                    for file_p in file_index {
+                        let file_p_2 = file_p.clone();
+                        let file = os.get(&object_store::path::Path::from(file_p)).await;
                         if let Ok(file) = file {
                             if let Ok(bytes) = file.bytes().await {
                                 yield Ok(bytes::Bytes::from(bytes)) as object_store::Result<bytes::Bytes>;
                             }
+                        } else {
+                            tracing::debug!("error getting file from store: {file_p_2}: {}", file.err().unwrap());
                         }
                     }
 
                     yield Ok(bytes::Bytes::from(logs))
                 };
                 return Some(Ok(Body::from_stream(stream)));
+            } else {
+                tracing::debug!("object store client not present, cannot stream logs from store");
             }
         }
     }
@@ -784,6 +792,7 @@ pub struct RunJobQuery {
     job_id: Option<Uuid>,
     tag: Option<String>,
     timeout: Option<i32>,
+    cache_ttl: Option<i32>,
 }
 
 impl RunJobQuery {
@@ -2880,13 +2889,16 @@ pub async fn run_wait_result_script_by_hash(
         tag,
         concurrent_limit,
         concurrency_time_window_s,
-        cache_ttl,
+        mut cache_ttl,
         language,
         dedicated_worker,
         priority,
         delete_after_use,
         timeout,
     ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
+    if let Some(run_query_cache_ttl) = run_query.cache_ttl {
+        cache_ttl = Some(run_query_cache_ttl);
+    }
     check_scopes(&authed, || format!("run:script/{path}"))?;
 
     let tag = run_query.tag.clone().or(tag);
@@ -3401,7 +3413,7 @@ pub async fn run_job_by_hash(
         tag,
         concurrent_limit,
         concurrency_time_window_s,
-        cache_ttl,
+        mut cache_ttl,
         language,
         dedicated_worker,
         priority,
@@ -3409,7 +3421,9 @@ pub async fn run_job_by_hash(
         timeout,
     ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
     check_scopes(&authed, || format!("run:script/{path}"))?;
-
+    if let Some(run_query_cache_ttl) = run_query.cache_ttl {
+        cache_ttl = Some(run_query_cache_ttl);
+    }
     let scheduled_for = run_query.get_scheduled_for(&db).await?;
     let tag = run_query.tag.clone().or(tag);
 

backend/windmill-api/src/lib.rs

@@ -107,11 +107,6 @@ lazy_static::lazy_static! {
         .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
         .build().unwrap();
 
-    pub static ref HTTP_CLIENT_11: reqwest_11::Client = reqwest_11::ClientBuilder::new()
-        .user_agent("windmill/beta")
-        .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
-        .build().unwrap();
-
     pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
         logins: HashMap::new(),
         connects: HashMap::new(),
@@ -171,18 +166,10 @@ pub async fn run_server(
 
     let sp_extension = Arc::new(saml_ee::build_sp_extension().await?);
 
-    let embeddings_db = if server_mode {
+    if server_mode {
         #[cfg(feature = "embedding")]
-        {
-            Some(load_embeddings_db(&db))
-        }
-        #[cfg(not(feature = "embedding"))]
-        {
-            Some(())
-        }
-    } else {
-        None
-    };
+        load_embeddings_db(&db)
+    }
 
     let job_helpers_service = {
         #[cfg(feature = "parquet")]
@@ -209,10 +196,7 @@ pub async fn run_server(
                         .nest("/apps", apps::workspaced_service())
                         .nest("/audit", audit::workspaced_service())
                         .nest("/capture", capture::workspaced_service())
-                        .nest(
-                            "/embeddings",
-                            embeddings::workspaced_service(embeddings_db.clone()),
-                        )
+                        .nest("/embeddings", embeddings::workspaced_service())
                         .nest("/drafts", drafts::workspaced_service())
                         .nest("/favorites", favorite::workspaced_service())
                         .nest("/flows", flows::workspaced_service())
@@ -250,10 +234,7 @@ pub async fn run_server(
                 .nest("/flows", flows::global_service())
                 .nest("/apps", apps::global_service().layer(cors.clone()))
                 .nest("/schedules", schedule::global_service())
-                .nest(
-                    "/embeddings",
-                    embeddings::global_service(embeddings_db.clone()),
-                )
+                .nest("/embeddings", embeddings::global_service())
                 .route_layer(from_extractor::<ApiAuthed>())
                 .route_layer(from_extractor::<users::Tokened>())
                 .nest("/jobs", jobs::global_root_service())
@@ -298,7 +279,8 @@ pub async fn run_server(
                 .route("/version", get(git_v))
                 .route("/uptodate", get(is_up_to_date))
                 .route("/ee_license", get(ee_license))
-                .route("/openapi.yaml", get(openapi)),
+                .route("/openapi.yaml", get(openapi))
+                .route("/openapi.json", get(openapi_json)),
         )
         .fallback(static_assets::static_handler)
         .layer(middleware_stack);
@@ -386,6 +368,10 @@ async fn openapi() -> &'static str {
     include_str!("../openapi-deref.yaml")
 }
 
+async fn openapi_json() -> &'static str {
+    include_str!("../openapi-deref.json")
+}
+
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
     db::migrate(db).await?;
     Ok(())

backend/windmill-api/src/schedule.rs

@@ -316,6 +316,7 @@ pub struct ListScheduleQuery {
     pub per_page: Option<usize>,
     pub path: Option<String>,
     pub is_flow: Option<bool>,
+    pub args: Option<String>,
 }
 
 async fn list_schedule(
@@ -339,6 +340,9 @@ async fn list_schedule(
     if let Some(is_flow) = lsq.is_flow {
         sqlb.and_where_eq("is_flow", "?".bind(&is_flow));
     }
+    if let Some(args) = &lsq.args {
+        sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
+    }
     let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
     let rows = sqlx::query_as::<_, Schedule>(&sql)
         .fetch_all(&mut *tx)

backend/windmill-api/src/scripts.rs

@@ -44,6 +44,7 @@ use windmill_common::{
     utils::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
+    HUB_BASE_URL,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 use windmill_parser_ts::remove_pinned_imports;
@@ -88,6 +89,8 @@ pub struct ScriptWDraft {
     pub timeout: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
 }
 
 pub fn global_service() -> Router {
@@ -288,7 +291,7 @@ async fn get_top_hub_scripts(
 
     let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
-        "https://hub.windmill.dev/scripts/top",
+        &format!("{}/scripts/top", *HUB_BASE_URL.read().await),
         Some(query_params),
         &db,
     )
@@ -472,8 +475,8 @@ async fn create_script(
          content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
          draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
          dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
-         delete_after_use, timeout, concurrency_key) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)",
+         delete_after_use, timeout, concurrency_key, visible_to_runner_only) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)",
         &w_id,
         &hash.0,
         ns.path,
@@ -501,6 +504,7 @@ async fn create_script(
         ns.delete_after_use,
         ns.timeout,
         ns.concurrency_key,
+        ns.visible_to_runner_only,
     )
     .execute(&mut tx)
     .await?;
@@ -720,7 +724,7 @@ async fn get_script_by_path_w_draft(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, ScriptWDraft>(
-        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key FROM script LEFT JOIN draft ON 
+        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only FROM script LEFT JOIN draft ON 
          script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
          WHERE script.path = $1 AND script.workspace_id = $2 \
          AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \

backend/windmill-api/src/settings.rs

@@ -26,7 +26,7 @@ use serde::Deserialize;
 use tokio::time::timeout;
 use windmill_common::{
     error::{self, to_anyhow, JsonResult, Result},
-    global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS},
+    global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS, HUB_BASE_URL_SETTING},
     server::Smtp,
 };
 
@@ -147,7 +147,6 @@ pub async fn test_s3_bucket(
     Ok("Tested blob storage successfully".to_string())
 }
 
-
 #[derive(Deserialize)]
 pub struct TestKey {
     pub license_key: String,
@@ -251,6 +250,7 @@ pub async fn get_global_setting(
     if !key.starts_with("default_error_handler_")
         && !key.starts_with("default_recovery_handler_")
         && key != AUTOMATE_USERNAME_CREATION_SETTING
+        && key != HUB_BASE_URL_SETTING
     {
         require_super_admin(&db, &authed.email).await?;
     }

backend/windmill-api/src/users.rs

@@ -1611,7 +1611,7 @@ async fn update_workspace_user(
     )
     .await?;
 
-    Ok(format!("user {username} updated"))
+    Ok(format!("user {} updated", user_email))
 }
 
 async fn update_user(

backend/windmill-api/src/workspaces.rs

@@ -112,7 +112,8 @@ pub fn workspaced_service() -> Router {
         .route("/leave", post(leave_workspace))
         .route("/get_workspace_name", get(get_workspace_name))
         .route("/change_workspace_name", post(change_workspace_name))
-        .route("/change_workspace_id", post(change_workspace_id));
+        .route("/change_workspace_id", post(change_workspace_id))
+        .route("/usage", get(get_usage));
 
     #[cfg(feature = "stripe")]
     {
@@ -146,7 +147,6 @@ struct Workspace {
     owner: String,
     deleted: bool,
     premium: bool,
-    is_overquota: bool,
 }
 
 #[derive(FromRow, Serialize, Debug)]
@@ -415,6 +415,30 @@ async fn edit_slack_command(
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
     let mut tx = db.begin().await?;
+
+    if es.slack_command_script.is_some() {
+        let exists_slack_command_with_team_id = sqlx::query_scalar!(
+            r#"
+                SELECT EXISTS (SELECT 1 
+                FROM workspace_settings 
+                WHERE workspace_id <> $1 
+                    AND slack_command_script IS NOT NULL
+                    AND slack_team_id IS NOT NULL 
+                    AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1))
+            "#,
+            &w_id
+        )
+        .fetch_one(&mut *tx)
+        .await?.unwrap_or(false);
+
+        if exists_slack_command_with_team_id {
+            return Err(Error::BadRequest(
+                "A workspace connected to the same slack team already has a command script. Please remove it first."
+                    .to_string(),
+            ));
+        }
+    }
+
     sqlx::query!(
         "UPDATE workspace_settings SET slack_command_script = $1 WHERE workspace_id = $2",
         es.slack_command_script,
@@ -2157,6 +2181,8 @@ struct ScriptMetadata {
     pub delete_after_use: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub restart_unless_cancelled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
 }
 
 pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -2432,6 +2458,7 @@ async fn tarball_workspace(
                 timeout: script.timeout,
                 delete_after_use: script.delete_after_use,
                 restart_unless_cancelled: script.restart_unless_cancelled,
+                visible_to_runner_only: script.visible_to_runner_only,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
             archive
@@ -2798,11 +2825,13 @@ async fn change_workspace_id(
 
     // duplicate workspace with new id name
     sqlx::query!(
-        "INSERT INTO workspace SELECT $1, $2, owner, deleted, premium, is_overquota FROM workspace WHERE id = $3",
+        "INSERT INTO workspace SELECT $1, $2, owner, deleted, premium FROM workspace WHERE id = $3",
         &rw.new_id,
         &rw.new_name,
         &old_id
-    ).execute(&mut *tx).await?;
+    )
+    .execute(&mut *tx)
+    .await?;
 
     sqlx::query!(
         "UPDATE account SET workspace_id = $1 WHERE workspace_id = $2",
@@ -3079,3 +3108,18 @@ async fn change_workspace_id(
         &old_id, &rw.new_id
     ))
 }
+
+async fn get_usage(Extension(db): Extension<DB>, Path(w_id): Path<String>) -> Result<String> {
+    let usage = sqlx::query_scalar!(
+        "
+    SELECT usage.usage FROM usage 
+    WHERE is_workspace = true 
+    AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
+    AND id = $1",
+        w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .unwrap_or(0);
+    Ok(usage.to_string())
+}

backend/windmill-common/src/flow_status.rs

@@ -137,8 +137,12 @@ pub enum FlowStatusModule {
         branch_chosen: Option<BranchChosen>,
         #[serde(skip_serializing_if = "Option::is_none")]
         branchall: Option<BranchAllStatus>,
+        #[serde(skip_serializing_if = "std::ops::Not::not")]
         #[serde(default = "default_false")]
         parallel: bool,
+        #[serde(skip_serializing_if = "std::ops::Not::not")]
+        #[serde(default = "default_false")]
+        while_loop: bool,
     },
     Success {
         id: String,

backend/windmill-common/src/flows.rs

@@ -44,6 +44,8 @@ pub struct Flow {
     pub ws_error_handler_muted: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub timeout: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
 }
 
 #[derive(Serialize, sqlx::FromRow)]
@@ -77,6 +79,7 @@ pub struct NewFlow {
     pub dedicated_worker: Option<bool>,
     pub timeout: Option<i32>,
     pub deployment_message: Option<String>,
+    pub visible_to_runner_only: Option<bool>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
@@ -102,6 +105,8 @@ pub struct FlowValue {
     #[serde(skip_serializing_if = "Option::is_none")]
     // Priority at the flow level
     pub priority: Option<i16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub concurrency_key: Option<String>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -200,6 +205,8 @@ pub struct Suspend {
     pub user_groups_required: Option<InputTransform>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub self_approval_disabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hide_cancel: Option<bool>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -235,6 +242,8 @@ pub struct FlowModule {
     pub priority: Option<i16>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub delete_after_use: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub continue_on_error: Option<bool>,
 }
 
 impl FlowModule {
@@ -309,6 +318,11 @@ pub enum FlowModuleValue {
         #[serde(skip_serializing_if = "Option::is_none")]
         parallelism: Option<u16>,
     },
+    WhileloopFlow {
+        modules: Vec<FlowModule>,
+        #[serde(default = "default_false")]
+        skip_failures: bool,
+    },
     BranchOne {
         branches: Vec<BranchOneModules>,
         default: Vec<FlowModule>,
@@ -386,6 +400,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec<FlowModule>) {
             timeout: None,
             priority: None,
             delete_after_use: None,
+            continue_on_error: None,
         });
     }
 }

backend/windmill-common/src/global_settings.rs

@@ -23,6 +23,7 @@ pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexistin
 pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config";
 
 pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation";
+pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
 
 pub const ENV_SETTINGS: [&str; 50] = [
     "DISABLE_NSJAIL",

backend/windmill-common/src/lib.rs

@@ -46,6 +46,8 @@ pub mod tracing_init;
 pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
 pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 4;
 
+pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
+
 lazy_static::lazy_static! {
     pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
     .ok()
@@ -69,6 +71,8 @@ lazy_static::lazy_static! {
 
     pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
     pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
+    pub static ref HUB_BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new(DEFAULT_HUB_BASE_URL.to_string()));
 }
 
 pub async fn shutdown_signal(

backend/windmill-common/src/s3_helpers.rs

@@ -377,7 +377,7 @@ pub async fn build_s3_client_from_settings(
         region,
         access_key,
         secret_key,
-        use_ssl: !settings.allow_http.unwrap_or(false),
+        use_ssl: !settings.allow_http.unwrap_or(true),
         path_style: None,
         port: None,
         token: None,

backend/windmill-common/src/scripts.rs

@@ -14,7 +14,7 @@ use std::{
 use crate::{
     error::{to_anyhow, Error},
     utils::http_get_from_hub,
-    DB,
+    DB, HUB_BASE_URL,
 };
 use serde::de::Error as _;
 use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
@@ -22,8 +22,7 @@ use serde_json::to_string_pretty;
 
 use crate::utils::StripPath;
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq)]
-#[derive(sqlx::Type)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq, sqlx::Type)]
 #[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
 #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
 pub enum ScriptLang {
@@ -62,13 +61,11 @@ impl ScriptLang {
     }
 }
 
-#[derive(PartialEq, Debug, Hash, Clone, Copy)]
-#[derive(sqlx::Type)]
+#[derive(PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)]
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
 
-#[derive(PartialEq)]
-#[derive(sqlx::Type)]
+#[derive(PartialEq, sqlx::Type)]
 #[sqlx(transparent, no_pg_array)]
 pub struct ScriptHashes(pub Vec<i64>);
 
@@ -109,8 +106,7 @@ impl Serialize for ScriptHashes {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, Hash)]
-#[derive(sqlx::Type)]
+#[derive(Serialize, Deserialize, Debug, Hash, sqlx::Type)]
 #[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")]
 #[serde(rename_all = "lowercase")]
 pub enum ScriptKind {
@@ -132,8 +128,7 @@ impl Display for ScriptKind {
     }
 }
 
-#[derive(Serialize)]
-#[derive(sqlx::FromRow)]
+#[derive(Serialize, sqlx::FromRow)]
 pub struct Script {
     pub workspace_id: String,
     pub hash: ScriptHash,
@@ -178,10 +173,11 @@ pub struct Script {
     pub restart_unless_cancelled: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub visible_to_runner_only: Option<bool>,
 }
 
-#[derive(Serialize)]
-#[derive(sqlx::FromRow)]
+#[derive(Serialize, sqlx::FromRow)]
 pub struct ListableScript {
     pub hash: ScriptHash,
     pub path: String,
@@ -212,8 +208,7 @@ pub struct ScriptHistoryUpdate {
     pub deployment_msg: Option<String>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-#[derive(sqlx::Type)]
+#[derive(Serialize, Deserialize, Debug, sqlx::Type)]
 #[sqlx(transparent)]
 #[serde(transparent)]
 pub struct Schema(pub serde_json::Value);
@@ -255,6 +250,7 @@ pub struct NewScript {
     pub deployment_message: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_key: Option<String>,
+    pub visible_to_runner_only: Option<bool>,
 }
 
 fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
@@ -357,7 +353,7 @@ pub async fn get_hub_script_by_path(
 
     let content = http_get_from_hub(
         http_client,
-        &format!("https://hub.windmill.dev/raw/{path}.ts"),
+        &format!("{}/raw/{}.ts", *HUB_BASE_URL.read().await, path),
         true,
         None,
         db,
@@ -381,7 +377,7 @@ pub async fn get_full_hub_script_by_path(
 
     let value = http_get_from_hub(
         http_client,
-        &format!("https://hub.windmill.dev/raw2/{path}"),
+        &format!("{}/raw2/{}", *HUB_BASE_URL.read().await, path),
         true,
         None,
         db,

backend/windmill-common/src/workspaces.rs

@@ -28,5 +28,6 @@ pub struct GitRepositorySettings {
     pub script_path: String,
     pub git_repo_resource_path: String,
     pub use_individual_branch: Option<bool>,
+    pub group_by_folder: Option<bool>,
     pub exclude_types_override: Option<Vec<ObjectType>>,
 }

backend/windmill-git-sync/src/lib.rs

@@ -15,7 +15,7 @@ pub mod git_sync_ee;
 pub use git_sync_ee::handle_deployment_metadata;
 pub type DB = Pool<Postgres>;
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum DeployedObject {
     Script { hash: ScriptHash, path: String, parent_path: Option<String> },
     Flow { path: String, parent_path: Option<String> },
@@ -47,7 +47,7 @@ impl DeployedObject {
 
     pub fn get_ignore_regex_filter(&self) -> bool {
         match self {
-            DeployedObject::User { .. } | DeployedObject::Group { .. } => true,
+            Self::User { .. } | Self::Group { .. } | Self::ResourceType { .. } => true,
             _ => false,
         }
     }

backend/windmill-queue/src/jobs.rs

@@ -21,7 +21,6 @@ use axum::{
     http::{request::Parts, Request, Uri},
     response::{IntoResponse, Response},
 };
-use bigdecimal::ToPrimitive;
 use chrono::{DateTime, Duration, Utc};
 use itertools::Itertools;
 #[cfg(feature = "prometheus")]
@@ -114,7 +113,7 @@ lazy_static::lazy_static! {
 #[cfg(feature = "enterprise")]
 const MAX_FREE_EXECS: i32 = 1000;
 #[cfg(feature = "enterprise")]
-const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
+const MAX_FREE_CONCURRENT_RUNS: i32 = 30;
 
 const ERROR_HANDLER_USERNAME: &str = "error_handler";
 const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
@@ -440,24 +439,6 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
     Ok(result)
 }
 
-fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
-    modules
-        .into_iter()
-        .filter_map(|m| match m {
-            FlowStatusModule::Success { job, flow_jobs, .. }
-            | FlowStatusModule::Failure { job, flow_jobs, .. } => {
-                if let Some(flow_jobs) = flow_jobs {
-                    Some(flow_jobs)
-                } else {
-                    Some(vec![job])
-                }
-            }
-            _ => None,
-        })
-        .flatten()
-        .collect::<Vec<_>>()
-}
-
 lazy_static::lazy_static! {
     pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
 }
@@ -486,32 +467,6 @@ pub async fn add_completed_job<
         ));
     }
 
-    let is_flow = queued_job.is_flow();
-    let duration = if is_flow {
-        let jobs = queued_job.parse_flow_status().map(|s| {
-            let mut modules = s.modules;
-            modules.extend([s.failure_module.module_status]);
-            flatten_jobs(modules)
-        });
-        if let Some(jobs) = jobs {
-            sqlx::query_scalar!(
-                "SELECT SUM(duration_ms) as duration FROM completed_job WHERE id = ANY($1)",
-                jobs.as_slice()
-            )
-            .fetch_one(db)
-            .await
-            .ok()
-            .flatten()
-            .map(|x| x.to_i64())
-            .flatten()
-        } else {
-            tracing::warn!("Could not parse flow status");
-            None
-        }
-    } else {
-        None
-    };
-
     let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
     let job_id = queued_job.id;
     // tracing::error!("1 {:?}", start.elapsed());
@@ -556,8 +511,8 @@ pub async fn add_completed_job<
                    , tag
                    , priority
                 )
-            VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($25, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30)
+            VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000, $7, $8, $9,\
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
          ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
         queued_job.workspace_id,
         queued_job.id,
@@ -583,7 +538,6 @@ pub async fn add_completed_job<
         queued_job.is_flow_step,
         skipped,
         queued_job.language.clone() as Option<ScriptLang>,
-        duration as Option<i64>,
         queued_job.email,
         queued_job.visible_to_owner,
         if mem_peak > 0 { Some(mem_peak) } else { None },
@@ -735,7 +689,7 @@ pub async fn add_completed_job<
     );
 
     #[cfg(feature = "enterprise")]
-    if *CLOUD_HOSTED && !is_flow && _duration > 1000 {
+    if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 {
         let additional_usage = _duration / 1000;
         let w_id = &queued_job.workspace_id;
         let premium_workspace =
@@ -745,14 +699,25 @@ pub async fn add_completed_job<
                 .map_err(|e| Error::InternalErr(format!("fetching if {w_id} is premium: {e}")))?;
         let _ = sqlx::query!(
                 "INSERT INTO usage (id, is_workspace, month_, usage) 
-                VALUES ($1, $2, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
-                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $3",
-                if premium_workspace { w_id } else { &queued_job.email },
-                premium_workspace,
+                VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) 
+                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+                w_id,
                 additional_usage as i32)
                 .execute(db)
                 .await
                 .map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
+
+        if !premium_workspace {
+            let _ = sqlx::query!(
+                "INSERT INTO usage (id, is_workspace, month_, usage) 
+                VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) 
+                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2",
+                queued_job.email,
+                additional_usage as i32)
+                .execute(db)
+                .await
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
+        }
     }
 
     if !skip_downstream_error_handlers
@@ -1036,7 +1001,7 @@ pub async fn send_error_to_workspace_handler<
     ).bind(&w_id)
     .fetch_optional(&mut *tx)
     .await
-    .context("sending error to global handler")?
+    .context("fetching error handler info from workspace_settings")?
     .ok_or_else(|| Error::InternalErr(format!("no workspace settings for id {w_id}")))?;
 
     if is_canceled && error_handler_muted_on_cancel {
@@ -1692,12 +1657,12 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             + Duration::try_seconds(i64::from(job_custom_concurrency_time_window_s))
                 .unwrap_or_default();
         tracing::info!("Job '{}' from path '{}' with concurrency key '{}' has reached its concurrency limit of {} jobs run in the last {} seconds. This job will be re-queued for next execution at {}", 
-            job_uuid, job_script_path, job_custom_concurrent_limit,  job_concurrency_key, job_custom_concurrency_time_window_s, estimated_next_schedule_timestamp);
+            job_uuid, job_script_path,  job_concurrency_key, job_custom_concurrent_limit, job_custom_concurrency_time_window_s, estimated_next_schedule_timestamp);
 
         let job_log_event = format!(
             "\nRe-scheduled job to {estimated_next_schedule_timestamp} due to concurrency limits with key {job_concurrency_key} and limit {job_custom_concurrent_limit} in the last {job_custom_concurrency_time_window_s} seconds",
         );
-        let _ = append_logs(job_uuid, pulled_job.workspace_id, job_log_event, db);
+        let _ = append_logs(job_uuid, pulled_job.workspace_id, job_log_event, db).await;
         if rsmq.is_some() {
             // if let Some(ref mut rsmq) = tx.rsmq {
             // if using redis, only one message at a time can be poped from the queue. Process only this message and move to the next elligible job
@@ -1919,50 +1884,63 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
 }
 
 async fn concurrency_key(db: &Pool<Postgres>, queued_job: &QueuedJob) -> String {
-    if queued_job.is_flow() {
-        // custom concurrency keys are not yet supported for flows
-        queued_job.full_path_with_workspace()
-    } else {
-        let concurrency_key = sqlx::query_scalar!(
-            "SELECT concurrency_key FROM script WHERE hash = $1",
-            queued_job.script_hash.unwrap_or(ScriptHash(0)).0
+    let r = if queued_job.is_flow() {
+        sqlx::query_scalar!(
+            "SELECT value->>'concurrency_key' FROM flow WHERE path = $1 AND workspace_id = $2",
+            queued_job.script_path,
+            queued_job.workspace_id
         )
         .fetch_one(db)
-        .await;
-        match concurrency_key {
-            Ok(Some(custom_concurrency_key)) => {
-                let workspaced =
-                    custom_concurrency_key.replace("$workspace", queued_job.workspace_id.as_str());
-                if RE_ARG_TAG.is_match(&workspaced) {
-                    let mut interpolated = workspaced.clone();
-                    for cap in RE_ARG_TAG.captures_iter(&workspaced) {
-                        let arg_name = cap.get(1).unwrap().as_str();
-                        let arg_value = match queued_job.args.as_ref() {
-                            Some(Json(args_map_json)) => match args_map_json.get(arg_name) {
-                                Some(arg_value_raw) => {
-                                    serde_json::to_string(arg_value_raw).unwrap_or_default()
-                                }
-                                None => "".to_string(),
-                            },
+        .await
+    } else {
+        sqlx::query_scalar!(
+            "SELECT concurrency_key FROM script WHERE hash = $1 AND workspace_id = $2",
+            queued_job.script_hash.unwrap_or(ScriptHash(0)).0,
+            queued_job.workspace_id
+        )
+        .fetch_one(db)
+        .await
+    };
+    process_custom_concurrency_key(queued_job, r).await
+}
+
+async fn process_custom_concurrency_key(
+    queued_job: &QueuedJob,
+    concurrency_key: Result<Option<String>, sqlx::Error>,
+) -> String {
+    match concurrency_key {
+        Ok(Some(custom_concurrency_key)) => {
+            let workspaced =
+                custom_concurrency_key.replace("$workspace", queued_job.workspace_id.as_str());
+            if RE_ARG_TAG.is_match(&workspaced) {
+                let mut interpolated = workspaced.clone();
+                for cap in RE_ARG_TAG.captures_iter(&workspaced) {
+                    let arg_name = cap.get(1).unwrap().as_str();
+                    let arg_value = match queued_job.args.as_ref() {
+                        Some(Json(args_map_json)) => match args_map_json.get(arg_name) {
+                            Some(arg_value_raw) => {
+                                serde_json::to_string(arg_value_raw).unwrap_or_default()
+                            }
                             None => "".to_string(),
-                        };
-                        interpolated = interpolated
-                            .replace(format!("$args[{}]", arg_name).as_str(), arg_value.as_str());
-                    }
-                    interpolated
-                } else {
-                    workspaced
+                        },
+                        None => "".to_string(),
+                    };
+                    interpolated = interpolated
+                        .replace(format!("$args[{}]", arg_name).as_str(), arg_value.as_str());
                 }
+                interpolated
+            } else {
+                workspaced
             }
-            Ok(None) => queued_job.full_path_with_workspace(),
-            _ => {
-                tracing::warn!(
-                    "Unable to retrieve concurrency key for script {:?} | {:?}",
-                    queued_job.script_path,
-                    queued_job.script_hash
-                );
-                queued_job.full_path_with_workspace()
-            }
+        }
+        Ok(None) => queued_job.full_path_with_workspace(),
+        _ => {
+            tracing::warn!(
+                "Unable to retrieve concurrency key for script {:?} | {:?}",
+                queued_job.script_path,
+                queued_job.script_hash
+            );
+            queued_job.full_path_with_workspace()
         }
     }
 }
@@ -2623,54 +2601,52 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
 ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
     #[cfg(feature = "enterprise")]
     if *CLOUD_HOSTED {
-        let row = sqlx::query!(
-            "SELECT premium, is_overquota FROM workspace WHERE id = $1",
-            workspace_id
-        )
-        .fetch_one(_db)
-        .await
-        .map_err(|e| {
-            Error::InternalErr(format!(
-                "fetching if {workspace_id} is premium and overquota: {e}"
-            ))
-        })?;
-        let premium_workspace = row.premium;
-        let is_overquota = premium_workspace && row.is_overquota;
+        let premium_workspace =
+            sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
+                .fetch_one(_db)
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!(
+                        "fetching if {workspace_id} is premium and overquota: {e}"
+                    ))
+                })?;
 
         // we track only non flow steps
-        let usage = if !matches!(
+        let (workspace_usage, user_usage) = if !matches!(
             job_payload,
             JobPayload::Flow { .. } | JobPayload::RawFlow { .. }
         ) {
-            sqlx::query_scalar!(
-                    "INSERT INTO usage (id, is_workspace, month_, usage) 
-                    VALUES ($1, $2, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 0) 
+            let workspace_usage = sqlx::query_scalar!(
+                    "INSERT INTO usage (id, is_workspace, month_, usage)
+                    VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)
                     ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 
                     RETURNING usage.usage",
-                    if premium_workspace { workspace_id } else { email },
-                    premium_workspace
+                    workspace_id
                 )
                 .fetch_one(_db)
                 .await
-                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?
-        } else if !premium_workspace {
-            sqlx::query_scalar!(
-                "
-        SELECT usage.usage + 1 FROM usage 
-        WHERE is_workspace = false AND
-     month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
-     AND id = $1",
-                email
-            )
-            .fetch_optional(_db)
-            .await?
-            .flatten()
-            .unwrap_or(0)
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?;
+
+            let user_usage = if !premium_workspace {
+                Some(sqlx::query_scalar!(
+                    "INSERT INTO usage (id, is_workspace, month_, usage)
+                    VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)
+                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 
+                    RETURNING usage.usage",
+                    email
+                )
+                .fetch_one(_db)
+                .await
+                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?)
+            } else {
+                None
+            };
+            (Some(workspace_usage), user_usage)
         } else {
-            0
+            (None, None)
         };
 
-        if is_overquota || !premium_workspace {
+        if !premium_workspace {
             let is_super_admin =
                 sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
                     .fetch_optional(_db)
@@ -2678,12 +2654,39 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                     .unwrap_or(false);
 
             if !is_super_admin {
-                if is_overquota {
-                    return Err(error::Error::BadRequest(format!(
-                        "Workspace {workspace_id} is overquota. Please update your subscription in the customer portal to continue using Windmill."
-                    )));
-                }
-                if usage > MAX_FREE_EXECS
+                let user_usage = if let Some(user_usage) = user_usage {
+                    user_usage
+                } else {
+                    sqlx::query_scalar!(
+                        "SELECT usage.usage + 1 FROM usage 
+                        WHERE is_workspace IS FALSE AND
+                        month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
+                        AND id = $1",
+                        email
+                    )
+                    .fetch_optional(_db)
+                    .await?
+                    .flatten()
+                    .unwrap_or(1)
+                };
+
+                let workspace_usage = if let Some(workspace_usage) = workspace_usage {
+                    workspace_usage
+                } else {
+                    sqlx::query_scalar!(
+                        "SELECT usage.usage + 1 FROM usage 
+                        WHERE is_workspace IS TRUE AND
+                        month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
+                        AND id = $1",
+                        workspace_id
+                    )
+                    .fetch_optional(_db)
+                    .await?
+                    .flatten()
+                    .unwrap_or(1)
+                };
+
+                if user_usage > MAX_FREE_EXECS
                     && !matches!(job_payload, JobPayload::Dependencies { .. })
                     && !matches!(job_payload, JobPayload::FlowDependencies { .. })
                     && !matches!(job_payload, JobPayload::AppDependencies { .. })
@@ -2692,6 +2695,16 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                     "User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                 )));
                 }
+                if workspace_id != "demo"
+                    && workspace_usage > MAX_FREE_EXECS
+                    && !matches!(job_payload, JobPayload::Dependencies { .. })
+                    && !matches!(job_payload, JobPayload::FlowDependencies { .. })
+                    && !matches!(job_payload, JobPayload::AppDependencies { .. })
+                {
+                    return Err(error::Error::BadRequest(format!(
+                    "Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
+                )));
+                }
                 let in_queue =
                     sqlx::query_scalar!("SELECT COUNT(id) FROM queue WHERE email = $1", email)
                         .fetch_one(_db)
@@ -2704,6 +2717,20 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                 )));
                 }
 
+                let in_queue_workspace = sqlx::query_scalar!(
+                    "SELECT COUNT(id) FROM queue WHERE workspace_id = $1",
+                    workspace_id
+                )
+                .fetch_one(_db)
+                .await?
+                .unwrap_or(0);
+
+                if in_queue_workspace > MAX_FREE_EXECS.into() {
+                    return Err(error::Error::BadRequest(format!(
+                    "Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
+                )));
+                }
+
                 let concurrent_runs = sqlx::query_scalar!(
                     "SELECT COUNT(id) FROM queue WHERE running = true AND email = $1",
                     email
@@ -2717,6 +2744,20 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                     "User {email} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
                 )));
                 }
+
+                let concurrent_runs_workspace = sqlx::query_scalar!(
+                    "SELECT COUNT(id) FROM queue WHERE running = true AND workspace_id = $1",
+                    workspace_id
+                )
+                .fetch_one(_db)
+                .await?
+                .unwrap_or(0);
+
+                if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS.into() {
+                    return Err(error::Error::BadRequest(format!(
+                    "Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
+                )));
+                }
             }
         }
     }
@@ -2963,6 +3004,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                     timeout: None,
                     priority: None,
                     delete_after_use: None,
+                    continue_on_error: None,
                 }],
                 same_worker: false,
                 failure_module: None,
@@ -2971,6 +3013,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                 skip_expr: None,
                 cache_ttl: cache_ttl.map(|val| val as u32),
                 early_return: None,
+                concurrency_key: None,
                 priority: priority,
             };
             (
@@ -3490,6 +3533,7 @@ async fn restarted_flows_resolution(
                                 len: branches.len(),
                             }),
                             parallel: parallel,
+                            while_loop: false,
                         });
                     }
                     FlowModuleValue::ForloopFlow { parallel, .. } => {
@@ -3522,6 +3566,7 @@ async fn restarted_flows_resolution(
                             branch_chosen: None,
                             branchall: None,
                             parallel: parallel,
+                            while_loop: false,
                         });
                     }
                     _ => {

backend/windmill-worker/nsjail/run.powershell.config.proto

@@ -107,6 +107,13 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "{JOB_DIR}/result2.out"
+    dst: "/tmp/result2.out"
+    rw: true
+    is_bind: true
+}
+
 iface_no_lo: true
 
 mount {

backend/windmill-worker/src/common.rs

@@ -635,14 +635,20 @@ async fn compact_logs(
     let extra_split = modulo < nlogs_len;
     let excess_size_modulo = if extra_split { nlogs_len - modulo } else { 0 };
     let excess_size = excess_size_modulo
-        + nlogs[excess_size_modulo..]
+        + nlogs
             .chars()
+            .skip(excess_size_modulo)
             .find_position(|x| x.is_line_break())
             .map(|(i, _)| i + 1)
             .unwrap_or(0);
 
     let (excess_prev_logs, current_logs) = if extra_split {
-        let (excess_prev_logs, current_logs) = nlogs.split_at(excess_size as usize);
+        let split_idx = nlogs
+            .char_indices()
+            .nth(excess_size_modulo)
+            .map(|(i, _)| i)
+            .unwrap_or(0);
+        let (excess_prev_logs, current_logs) = nlogs.split_at(split_idx);
         (excess_prev_logs, current_logs.to_string())
     } else {
         ("", nlogs.to_string())
@@ -715,18 +721,16 @@ async fn default_disk_log_storage(
                     e
                 })
                 .ok();
-            let created = tokio::fs::File::create(&path)
-                .await;
+            let created = tokio::fs::File::create(&path).await;
             if let Err(e) = created {
                 tracing::error!("Could not create logs file {path}: {e:?}",);
-                return
+                return;
             }
             if let Err(e) = tokio::fs::write(&path, prev_logs).await {
                 tracing::error!("Could not write to logs file {path}: {e:?}");
             } else {
                 tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
             }
-
         }
     }
 }

backend/windmill-worker/src/global_cache.rs

@@ -1,5 +1,5 @@
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
-use crate::{PIP_CACHE_DIR, ROOT_CACHE_DIR};
+use crate::PIP_CACHE_DIR;
 
 // #[cfg(feature = "enterprise")]
 // use rand::Rng;
@@ -74,38 +74,35 @@ pub async fn build_tar_and_push(
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
 pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> error::Result<()> {
     use object_store::path::Path;
-    use tokio::fs::metadata;
     let folder_name = folder.split("/").last().unwrap();
 
     tracing::info!("Attempting to pull piptar {folder_name} from bucket");
 
     let start = Instant::now();
     let tar_path = format!("tar/pip/{folder_name}.tar");
-    let target = format!("{ROOT_CACHE_DIR}/{tar_path}.single");
 
-    let object = client
-        .get(&Path::from(format!("tar/pip/{folder_name}.tar")))
-        .await;
+    let object = client.get(&Path::from(tar_path.clone())).await;
     if let Err(e) = object {
-        tracing::info!("Failed to put tar to s3: {tar_path}. Error: {:?}", e);
+        tracing::info!("Failed to pull tar from s3: {tar_path}. Error: {:?}", e);
         return Err(error::Error::ExecutionErr(format!(
-            "Failed to put tar to s3: {tar_path}"
+            "Failed to pull tar from s3: {tar_path}"
         )));
     }
 
-    use tokio::io::AsyncWriteExt;
-    let mut file = tokio::fs::OpenOptions::new()
-        .write(true)
-        .create(true)
-        .open(&target)
-        .await
-        .unwrap();
+    let bytes = object.unwrap().bytes().await;
+    if bytes.is_err() {
+        tracing::info!(
+            "Failed to read tar from s3: {tar_path}. Error: {:?}",
+            bytes.err()
+        );
+        return Err(error::Error::ExecutionErr(format!(
+            "Failed to read tar from s3: {tar_path}"
+        )));
+    }
+    let bytes = bytes.unwrap();
+    tracing::info!("{tar_path} len: {}", bytes.len());
 
-    file.write_all(&object.unwrap().bytes().await.unwrap())
-        .await
-        .unwrap();
-
-    if metadata(&target).await.is_err() {
+    if bytes.len() == 0 {
         tracing::info!(
             "piptar {folder_name} not found in bucket. Took {:?}ms",
             start.elapsed().as_millis()
@@ -116,15 +113,11 @@ pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> erro
     }
     // tracing::info!("B: {target} {folder}");
 
-    extract_pip_tar(&target, &folder).await.map_err(|e| {
+    extract_pip_tar(bytes, &folder).await.map_err(|e| {
         tracing::error!("Failed to extract piptar {folder_name}. Error: {:?}", e);
         e
     })?;
 
-    tokio::fs::remove_file(&target).await.map_err(|e| {
-        tracing::error!("Failed to remove piptar {folder_name}. Error: {:?}", e);
-        e
-    })?;
     tracing::info!(
         "Finished pulling and extracting {folder_name}. Took {:?}ms",
         start.elapsed().as_millis()
@@ -134,16 +127,14 @@ pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> erro
 }
 
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
-pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
+pub async fn extract_pip_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
+    use bytes::Buf;
     use tokio::fs::{self};
 
     let start: Instant = Instant::now();
     fs::create_dir_all(&folder).await?;
 
-    let mut ar = tar::Archive::new(
-        std::fs::File::open(tar)
-            .map_err(|e| error::Error::ExecutionErr(format!("Failed to open tar {tar}: {e}")))?,
-    );
+    let mut ar = tar::Archive::new(tar.reader());
 
     if let Err(e) = ar.unpack(folder) {
         tracing::info!("Failed to untar piptar. Error: {:?}", e);

backend/windmill-worker/src/worker.rs

@@ -1162,6 +1162,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                     token,
                 } => {
                     // let r;
+                    tracing::info!(parent_flow = %flow, "updating flow status");
                     if let Err(e) = update_flow_status_after_job_completion(
                         &db2,
                         &AuthedClient {
@@ -1185,7 +1186,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                     )
                     .await
                     {
-                        tracing::error!("Error updating flow status after job completion: {e}");
+                        tracing::error!("Error updating flow status after job completion for {flow} on {worker_name2}: {e}");
                     }
                 }
                 SendResult::Kill => {
@@ -1337,6 +1338,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
             None
         }
     };
+    let mut suspend_first_success = false;
 
     loop {
         #[cfg(feature = "benchmark")]
@@ -1426,19 +1428,20 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         r
                     }
                 },
-                (job, timer) = async {
+                (job, timer, suspend_first) = async {
                     let pull_time = Instant::now();
-                    let suspend_first = if last_checked_suspended.elapsed().as_secs() > 3 {
+                    let suspend_first = if suspend_first_success || last_checked_suspended.elapsed().as_secs() > 3 {
                         last_checked_suspended = Instant::now();
                         true
                     } else { false };
-                    pull(&db, rsmq.clone(), suspend_first).map(|x| (x, pull_time)).await
+                    pull(&db, rsmq.clone(), suspend_first).map(|x| (x, pull_time, suspend_first)).await
                 } => {
                     add_time!(timing, loop_start, "post pull");
                     // tracing::debug!("pulled job: {:?}", job.as_ref().ok().and_then(|x| x.as_ref().map(|y| y.id)));
                     let duration_pull_s = timer.elapsed().as_secs_f64();
                     let err_pull = job.is_ok();
                     let empty = job.as_ref().is_ok_and(|x| x.is_none());
+                    suspend_first_success = suspend_first && !empty;
                     if !agent_mode && duration_pull_s > 0.5 {
                         tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
                         #[cfg(feature = "prometheus")]
@@ -1863,6 +1866,22 @@ async fn spawn_dedicated_workers_for_flow(
                 .await;
                 workers.extend(w);
             }
+            FlowModuleValue::WhileloopFlow { modules, .. } => {
+                let w = spawn_dedicated_workers_for_flow(
+                    &modules,
+                    path,
+                    w_id,
+                    killpill_tx.clone(),
+                    killpill_rx,
+                    db,
+                    worker_dir,
+                    base_internal_url,
+                    worker_name,
+                    job_completed_tx,
+                )
+                .await;
+                workers.extend(w);
+            }
             FlowModuleValue::BranchOne { branches, default } => {
                 for modules in branches
                     .iter()
@@ -2243,6 +2262,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
                 let timer = _worker_flow_transition_duration
                     .as_ref()
                     .map(|x| x.start_timer());
+                tracing::info!(parent_flow = %parent_job, subflow = %job.id, "updating flow status (2)");
                 update_flow_status_after_job_completion(
                     db,
                     client,
@@ -2280,6 +2300,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
         .await?;
         if job.is_flow_step {
             if let Some(parent_job) = job.parent_job {
+                tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status");
                 update_flow_status_after_job_completion(
                     db,
                     client,
@@ -2393,6 +2414,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
         };
 
         let wrapped_error = WrappedError { error: err.clone() };
+        tracing::error!(parent_flow = %flow, subflow = %job_status_to_update, "handle job error, updating flow status: {err:?}");
         let updated_flow = update_flow_status_after_job_completion(
             db,
             client,

backend/windmill-worker/src/worker_flow.rs

@@ -116,6 +116,7 @@ pub async fn update_flow_status_after_job_completion<
         {
             Ok(j) => j,
             Err(e) => {
+                tracing::error!("Error while updating flow status of {} after  completion of {}, updating flow status again with error: {e}", nrec.flow,&nrec.job_id_for_status);
                 update_flow_status_after_job_completion_internal(
                     db,
                     client,
@@ -152,6 +153,7 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
 pub struct SkipIfStopped {
     pub skip_if_stopped: Option<bool>,
     pub stop_early_expr: Option<String>,
+    pub continue_on_error: Option<bool>,
     pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
 }
 
@@ -232,24 +234,27 @@ pub async fn update_flow_status_after_job_completion_internal<
         let is_failure_step =
             old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0;
 
-        let (mut stop_early, skip_if_stop_early) = if let Some(se) = stop_early_override {
+        let (mut stop_early, skip_if_stop_early, continue_on_error) = if let Some(se) =
+            stop_early_override
+        {
             //do not stop early if module is a flow step
             let flow_job = get_queued_job(&flow, w_id, db)
                 .await?
                 .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
             let module = get_module(&flow_job, module_index);
             if module.is_some_and(|x| matches!(x.value, FlowModuleValue::Flow { .. })) {
-                (false, false)
+                (false, false, false)
             } else {
-                (true, se)
+                (true, se, false)
             }
         } else if is_failure_step {
-            (false, false)
+            (false, false, false)
         } else {
             let row = sqlx::query(
             "SELECT 
                     raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,
                     (raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,
+                    (raw_flow->'modules'->$1::int->'continue_on_error')::bool as continue_on_error,
                     args 
                 FROM queue
                 WHERE id = $2"
@@ -280,7 +285,11 @@ pub async fn update_flow_status_after_job_completion_internal<
                 } else {
                     false
                 };
-            (stop_early, r.skip_if_stopped.unwrap_or(false))
+            (
+                stop_early,
+                r.skip_if_stopped.unwrap_or(false),
+                r.continue_on_error.unwrap_or(false),
+            )
         };
 
         let skip_branch_failure = match module_status {
@@ -314,8 +323,17 @@ pub async fn update_flow_status_after_job_completion_internal<
                             flow
                         )
                         .fetch_one(&mut tx)
-                        .await?
+                        .await.map_err(|e| {
+                            Error::InternalErr(format!(
+                                "error while fetching iterator index: {e}"
+                            ))
+                        })?
                         .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?;
+                        tracing::info!(
+                            "parallel iteration {job_id_for_status} of flow {flow} update nindex: {nindex} len: {len}",
+                            nindex = nindex,
+                            len = itered.len()
+                        );
                         (nindex, itered.len() as i32)
                     }
                     (_, Some(BranchAllStatus { len, .. })) => {
@@ -328,7 +346,12 @@ pub async fn update_flow_status_after_job_completion_internal<
                             flow
                         )
                         .fetch_one(&mut tx)
-                        .await?
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!(
+                                "error while fetching branchall index: {e}"
+                            ))
+                        })?
                         .ok_or_else(|| Error::InternalErr(format!("requiring an index in InProgress")))?;
                         (nindex, *len as i32)
                     }
@@ -343,7 +366,12 @@ pub async fn update_flow_status_after_job_completion_internal<
                             jobs.as_slice()
                         )
                         .fetch_all(&mut tx)
-                        .await?
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!(
+                                "error while fetching sucess from completed_jobs: {e}"
+                            ))
+                        })?
                         .into_iter()
                         .all(|x| x)
                     {
@@ -367,25 +395,49 @@ pub async fn update_flow_status_after_job_completion_internal<
                     let r = sqlx::query_scalar!(
                         "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 RETURNING last_ping",
                         flow,
-                    ).fetch_optional(db).await?;
+                    ).fetch_optional(db).await.map_err(|e| {
+                        Error::InternalErr(format!(
+                            "error while deleting parallel_monitor_lock: {e}"
+                        ))
+                    })?;
                     if r.is_some() {
                         tracing::info!(
                             "parallel flow has removed lock on its parent, last ping was {:?}",
                             r.unwrap()
                         );
                     }
+                    tracing::info!(
+                        "parallel iteration {job_id_for_status} of flow {flow} has finished",
+                    );
 
                     (true, Some(new_status))
                 } else {
                     tx.commit().await?;
 
                     if parallelism.is_some() {
-                        sqlx::query!(
-                            "UPDATE queue SET suspend = suspend - 1 WHERE parent_job = $1",
+                        // this ensure that the lock is taken in the same order and thus avoid deadlocks
+                        let ids = sqlx::query_scalar!(
+                            "SELECT id FROM queue WHERE parent_job = $1 AND suspend > 0 ORDER by suspend",
                             flow
                         )
-                        .execute(db)
-                        .await?;
+                        .fetch_all(db)
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!("error while locking jobs to decrease parallelism of: {e}"))
+                        })?;
+                        for id in ids {
+                            sqlx::query!(
+                                "UPDATE queue SET suspend = suspend - 1 WHERE id = $1 AND suspend > 0",
+                                id
+                            )
+                            .execute(db)
+                            .await
+                            .map_err(|e| {
+                                Error::InternalErr(format!(
+                                    "error decreasing suspend for {id}: {e}"
+                                ))
+                            })?;
+                        }
                     }
 
                     sqlx::query!(
@@ -395,13 +447,18 @@ pub async fn update_flow_status_after_job_completion_internal<
                         flow
                     )
                     .execute(db)
-                    .await?;
+                    .await
+                    .map_err(|e| {
+                        Error::InternalErr(format!("error while setting last ping to null: {e}"))
+                    })?;
 
                     let r = sqlx::query_scalar!(
                         "DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping",
                         flow,
                         job_id_for_status
-                    ).fetch_optional(db).await?;
+                    ).fetch_optional(db).await.map_err(|e| {
+                        Error::InternalErr(format!("error while removing parallel_monitor_lock: {e}"))
+                    })?;
                     if r.is_some() {
                         tracing::info!(
                             "parallel flow has removed lock on its parent, last ping was {:?}",
@@ -414,8 +471,12 @@ pub async fn update_flow_status_after_job_completion_internal<
             }
             FlowStatusModule::InProgress {
                 iterator: Some(windmill_common::flow_status::Iterator { index, itered, .. }),
+                while_loop,
                 ..
-            } if (*index + 1 < itered.len() && (success || skip_loop_failures)) && !stop_early => {
+            } if (*while_loop
+                || (*index + 1 < itered.len()) && (success || skip_loop_failures))
+                && !stop_early =>
+            {
                 (false, None)
             }
             FlowStatusModule::InProgress {
@@ -457,8 +518,30 @@ pub async fn update_flow_status_after_job_completion_internal<
                         }),
                     )
                 } else {
+                    let inc = if continue_on_error {
+                        let retry = sqlx::query_scalar!(
+                            "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
+                            flow,
+                            old_status.step
+                        )
+                        .fetch_optional(&mut tx)
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!("error while getting retry fromn step: {e}"))
+                        })?
+                        .flatten();
+
+                        let retry = retry
+                            .map(|x| serde_json::from_value::<Retry>(x).ok())
+                            .flatten()
+                            .unwrap_or_default();
+                        tracing::error!("UPDATE FLOW STATUS 2: {retry:#?} ");
+                        next_retry(&retry, &old_status.retry).is_none()
+                    } else {
+                        false
+                    };
                     (
-                        false,
+                        inc,
                         Some(FlowStatusModule::Failure {
                             id: module_status.id(),
                             job: job_id_for_status.clone(),
@@ -479,7 +562,10 @@ pub async fn update_flow_status_after_job_completion_internal<
                 flow
             )
             .execute(&mut tx)
-            .await?;
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!("error while setting flow index for {flow}: {e}"))
+            })?;
             old_status.step + 1
         } else {
             old_status.step
@@ -497,7 +583,11 @@ pub async fn update_flow_status_after_job_completion_internal<
                     flow
                 )
                 .fetch_one(&mut tx)
-                .await?;
+                .await.map_err(|e| {
+                    Error::InternalErr(format!(
+                        "error while fetching failure module: {e}"
+                    ))
+                })?;
 
                 sqlx::query!(
                     "UPDATE queue
@@ -510,7 +600,12 @@ pub async fn update_flow_status_after_job_completion_internal<
                     flow
                 )
                 .execute(&mut tx)
-                .await?;
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!(
+                        "error while setting flow status in failure step: {e}"
+                    ))
+                })?;
             } else {
                 sqlx::query!(
                     "UPDATE queue
@@ -521,7 +616,10 @@ pub async fn update_flow_status_after_job_completion_internal<
                     flow
                 )
                 .execute(&mut tx)
-                .await?;
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!("error while setting new flow status: {e}"))
+                })?;
 
                 if let Some(job_result) = new_status.job_result() {
                     sqlx::query!(
@@ -533,7 +631,11 @@ pub async fn update_flow_status_after_job_completion_internal<
                         flow
                     )
                     .execute(&mut tx)
-                    .await?;
+                    .await.map_err(|e| {
+                        Error::InternalErr(format!(
+                            "error while setting leaf jobs: {e}"
+                        ))
+                    })?;
                 }
             }
         }
@@ -546,7 +648,9 @@ pub async fn update_flow_status_after_job_completion_internal<
             _ => result.to_owned(),
         };
 
-        if matches!(&new_status, Some(FlowStatusModule::Success { .. })) {
+        if old_status.retry.fail_count > 0
+            && matches!(&new_status, Some(FlowStatusModule::Success { .. }))
+        {
             sqlx::query(
                 "UPDATE queue
                 SET flow_status = flow_status - 'retry'
@@ -567,7 +671,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             .root_job
             .map(|x| x.to_string())
             .unwrap_or_else(|| "none".to_string());
-        tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
+        tracing::info!(id = %flow_job.id, root_id = %job_root, worker_name = %worker_name, "update flow status");
 
         let module = get_module(&flow_job, module_index);
 
@@ -579,7 +683,9 @@ pub async fn update_flow_status_after_job_completion_internal<
             _ if flow_job.canceled => false,
             true => !is_last_step,
             false if unrecoverable => false,
-            false if skip_branch_failure || skip_loop_failures => !is_last_step,
+            false if skip_branch_failure || skip_loop_failures || continue_on_error => {
+                !is_last_step
+            }
             false
                 if next_retry(
                     &module.and_then(|m| m.retry.clone()).unwrap_or_default(),
@@ -641,7 +747,10 @@ pub async fn update_flow_status_after_job_completion_internal<
                     &_cleanup_module.flow_jobs_to_clean,
                 )
                 .execute(db)
-                .await?;
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!("error while cleaning up completed_job: {e}"))
+                })?;
             }
         }
         if flow_job.canceled {
@@ -760,6 +869,8 @@ pub async fn update_flow_status_after_job_completion_internal<
         }
 
         if let Some(parent_job) = flow_job.parent_job {
+            tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, worker_name = %worker_name, "subflow is finished, updating parent flow status");
+
             return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
                 flow: parent_job,
                 job_id_for_status: flow,
@@ -1634,10 +1745,17 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         }
     };
 
+    let retry = if matches!(&status_module, FlowStatusModule::Failure { .. },) {
+        let retry = &module.retry.clone().unwrap_or_default();
+        next_retry(retry, &status.retry)
+    } else {
+        None
+    };
     let get_args_from_id = match &status_module {
-        FlowStatusModule::Failure { job, .. } => {
-            let retry = &module.retry.clone().unwrap_or_default();
-            if let Some((fail_count, retry_in)) = next_retry(retry, &status.retry) {
+        FlowStatusModule::Failure { job, .. }
+            if retry.as_ref().is_some() || !module.continue_on_error.is_some_and(|x| x) =>
+        {
+            if let Some((fail_count, retry_in)) = retry {
                 tracing::debug!(
                     retry_in_seconds = retry_in.as_secs(),
                     fail_count = fail_count,
@@ -1881,7 +1999,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 ..
             } => args.as_ref().map(|args| args.clone()),
             NextStatus::NextLoopIteration {
-                next: NextIteration { new_args, .. },
+                next: ForloopNextIteration { new_args, .. },
                 simple_input_transforms,
             } => {
                 let mut args = if let Ok(args) = args.as_ref() {
@@ -2049,7 +2167,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
     let first_uuid = uuids[0];
     let new_status = match next_status {
         NextStatus::NextLoopIteration {
-            next: NextIteration { index, itered, mut flow_jobs, .. },
+            next: ForloopNextIteration { index, itered, mut flow_jobs, while_loop, .. },
             ..
         } => {
             let uuid = one_uuid?;
@@ -2064,6 +2182,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 branchall: None,
                 id: status_module.id(),
                 parallel: false,
+                while_loop,
             }
         }
         NextStatus::AllFlowJobs { iterator, branchall, .. } => FlowStatusModule::InProgress {
@@ -2074,6 +2193,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             branchall,
             id: status_module.id(),
             parallel: true,
+            while_loop: false,
         },
         NextStatus::NextBranchStep(NextBranch { mut flow_jobs, status, .. }) => {
             let uuid = one_uuid?;
@@ -2087,6 +2207,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 branchall: Some(status),
                 id: status_module.id(),
                 parallel: false,
+                while_loop: false,
             }
         }
 
@@ -2098,6 +2219,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             branchall: None,
             id: status_module.id(),
             parallel: false,
+            while_loop: false,
         },
         NextStatus::NextStep => {
             FlowStatusModule::WaitingForExecutor { id: status_module.id(), job: one_uuid? }
@@ -2231,16 +2353,17 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
 /// Some state about the current/last forloop FlowStatusModule used to initialized the next
 /// iteration's FlowStatusModule after pushing a job
 #[derive(Debug)]
-struct NextIteration {
+struct ForloopNextIteration {
     index: usize,
     itered: Vec<serde_json::Value>,
     flow_jobs: Vec<Uuid>,
     new_args: Iter,
+    while_loop: bool,
 }
 
-enum LoopStatus {
+enum ForLoopStatus {
     ParallelIteration { itered: Vec<serde_json::Value> },
-    NextIteration(NextIteration),
+    NextIteration(ForloopNextIteration),
     EmptyIterator,
 }
 
@@ -2256,7 +2379,7 @@ enum NextStatus {
     BranchChosen(BranchChosen),
     NextBranchStep(NextBranch),
     NextLoopIteration {
-        next: NextIteration,
+        next: ForloopNextIteration,
         simple_input_transforms: Option<HashMap<String, InputTransform>>,
     },
     AllFlowJobs {
@@ -2391,200 +2514,74 @@ async fn compute_next_flow_transform(
                 NextStatus::NextStep,
             ))
         }
+        FlowModuleValue::WhileloopFlow { modules, .. } => {
+            // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
+            let is_simple = is_simple_modules(modules, flow);
+            let flow_jobs = match status_module {
+                FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. } => {
+                    flow_jobs.clone()
+                }
+                _ => vec![],
+            };
+            let next_loop_idx = flow_jobs.len();
+            next_loop_iteration(
+                flow,
+                status,
+                ForloopNextIteration {
+                    index: next_loop_idx,
+                    itered: vec![],
+                    flow_jobs: flow_jobs.clone(),
+                    new_args: Iter { index: next_loop_idx as i32, value: json!(next_loop_idx) },
+                    while_loop: true,
+                },
+                modules,
+                flow_job,
+                is_simple,
+                db,
+                module,
+                delete_after_use,
+            )
+            .await
+        }
         /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
         FlowModuleValue::ForloopFlow { modules, iterator, parallel, .. } => {
             // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
-            let is_simple = modules.len() == 1
-                && modules[0].value.is_simple()
-                && modules[0].sleep.is_none()
-                && modules[0].suspend.is_none()
-                && modules[0].cache_ttl.is_none()
-                && (modules[0].mock.is_none()
-                    && modules[0].mock.as_ref().is_some_and(|m| !m.enabled)
-                    && flow.failure_module.is_none());
+            let is_simple = is_simple_modules(modules, flow);
 
-            let next_loop_status = match status_module {
-                FlowStatusModule::WaitingForPriorSteps { .. }
-                | FlowStatusModule::WaitingForEvents { .. }
-                | FlowStatusModule::WaitingForExecutor { .. } => {
-                    let by_id = if let Some(x) = by_id {
-                        x
-                    } else {
-                        get_transform_context(&flow_job, previous_id, &status).await?
-                    };
-                    /* Iterator is an InputTransform, evaluate it into an array. */
-                    let itered_raw = match iterator {
-                        InputTransform::Static { value } => to_raw_value(value),
-                        InputTransform::Javascript { expr } => {
-                            let mut context = HashMap::with_capacity(5);
-                            context.insert("result".to_string(), arc_last_job_result.clone());
-                            context.insert("previous_result".to_string(), arc_last_job_result);
-                            context.insert("resumes".to_string(), resumes);
-                            context.insert("resume".to_string(), resume);
-                            context.insert("approvers".to_string(), approvers);
-
-                            eval_timeout(
-                                expr.to_string(),
-                                context,
-                                Some(arc_flow_job_args),
-                                Some(client),
-                                Some(by_id),
-                            )
-                            .await?
-                        }
-                    };
-                    let itered = serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get())
-                        .map_err(|not_array| {
-                        Error::ExecutionErr(format!("Expected an array value in the iterator expression, found: {not_array}"))
-                    })?;
-
-                    if itered.is_empty() {
-                        LoopStatus::EmptyIterator
-                    } else if *parallel {
-                        LoopStatus::ParallelIteration { itered }
-                    } else if let Some(first) = itered.first() {
-                        let iter = Iter { index: 0 as i32, value: first.to_owned() };
-                        LoopStatus::NextIteration(NextIteration {
-                            index: 0,
-                            itered,
-                            flow_jobs: vec![],
-                            new_args: iter,
-                        })
-                    } else {
-                        panic!("itered cannot be empty")
-                    }
-                }
-
-                FlowStatusModule::InProgress {
-                    iterator: Some(windmill_common::flow_status::Iterator { itered, index }),
-                    flow_jobs: Some(flow_jobs),
-                    ..
-                } if !*parallel => {
-                    let itered_new = if itered.is_empty() {
-                        // it's possible we need to re-compute the iterator Input Transforms here, in particular if the flow is being restarted inside the loop
-                        let by_id = if let Some(x) = by_id {
-                            x
-                        } else {
-                            get_transform_context(&flow_job, previous_id, &status).await?
-                        };
-                        let itered_raw = match iterator {
-                            InputTransform::Static { value } => to_raw_value(value),
-                            InputTransform::Javascript { expr } => {
-                                let mut context = HashMap::with_capacity(5);
-                                context.insert("result".to_string(), arc_last_job_result.clone());
-                                context.insert("previous_result".to_string(), arc_last_job_result);
-                                context.insert("resumes".to_string(), resumes);
-                                context.insert("resume".to_string(), resume);
-                                context.insert("approvers".to_string(), approvers);
-
-                                eval_timeout(
-                                    expr.to_string(),
-                                    context,
-                                    Some(arc_flow_job_args),
-                                    Some(client),
-                                    Some(by_id),
-                                )
-                                .await?
-                            }
-                        };
-                        serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
-                            |not_array| {
-                                Error::ExecutionErr(format!(
-                                    "Expected an array value, found: {not_array}"
-                                ))
-                            },
-                        )?
-                    } else {
-                        itered.clone()
-                    };
-                    let (index, next) = index
-                        .checked_add(1)
-                        .and_then(|i| itered_new.get(i).map(|next| (i, next)))
-                        .with_context(|| {
-                            format!("Could not find iteration number {index} restarting inside the for-loop flow. It's possible the itered-array has changed and this value isn't available anymore.")
-                        })?;
-
-                    LoopStatus::NextIteration(NextIteration {
-                        index,
-                        itered: itered_new.clone(),
-                        flow_jobs: flow_jobs.clone(),
-                        new_args: Iter { index: index as i32, value: next.to_owned() },
-                    })
-                }
-
-                _ => Err(Error::BadRequest(format!(
-                    "Unrecognized module status for ForloopFlow {status_module:?}"
-                )))?,
-            };
+            let next_loop_status = next_forloop_status(
+                status_module,
+                by_id,
+                flow_job,
+                previous_id,
+                status,
+                iterator,
+                arc_last_job_result,
+                resumes,
+                resume,
+                approvers,
+                arc_flow_job_args,
+                client,
+                parallel,
+            )
+            .await?;
 
             match next_loop_status {
-                LoopStatus::EmptyIterator => Ok(NextFlowTransform::EmptyInnerFlows),
-                LoopStatus::NextIteration(ns) => {
-                    let mut fm = flow.failure_module.clone();
-                    if let Some(mut failure_module) = flow.failure_module.clone() {
-                        failure_module.id_append(&format!("{}/{}", status.step, ns.index));
-                        fm = Some(failure_module);
-                    }
-                    let mut modules = (*modules).clone();
-                    add_virtual_items_if_necessary(&mut modules);
-                    let inner_path = Some(format!("{}/loop-{}", flow_job.script_path(), ns.index));
-                    if is_simple {
-                        let payload = payload_from_simple_module(
-                            &modules[0].value,
-                            db,
-                            flow_job,
-                            module,
-                            inner_path,
-                        )
-                        .await?;
-                        Ok(NextFlowTransform::Continue(
-                            ContinuePayload::SingleJob(payload),
-                            NextStatus::NextLoopIteration {
-                                next: ns,
-                                simple_input_transforms: if is_simple {
-                                    match &modules[0].value {
-                                        FlowModuleValue::Script { input_transforms, .. }
-                                        | FlowModuleValue::RawScript { input_transforms, .. }
-                                        | FlowModuleValue::Flow { input_transforms, .. } => {
-                                            Some(input_transforms.clone())
-                                        }
-                                        _ => None,
-                                    }
-                                } else {
-                                    None
-                                },
-                            },
-                        ))
-                    } else {
-                        Ok(NextFlowTransform::Continue(
-                            ContinuePayload::SingleJob(JobPayloadWithTag {
-                                payload: JobPayload::RawFlow {
-                                    value: FlowValue {
-                                        modules,
-                                        failure_module: fm,
-                                        same_worker: flow.same_worker,
-                                        concurrent_limit: None,
-                                        concurrency_time_window_s: None,
-                                        skip_expr: None,
-                                        cache_ttl: None,
-                                        priority: None,
-                                        early_return: None,
-                                    },
-                                    path: inner_path,
-                                    restarted_from: None,
-                                },
-                                tag: None,
-                                delete_after_use: delete_after_use,
-                                timeout: None,
-                            }),
-                            NextStatus::NextLoopIteration {
-                                next: ns,
-                                simple_input_transforms: None,
-                            },
-                        ))
-                    }
+                ForLoopStatus::EmptyIterator => Ok(NextFlowTransform::EmptyInnerFlows),
+                ForLoopStatus::NextIteration(ns) => {
+                    next_loop_iteration(
+                        flow,
+                        status,
+                        ns,
+                        modules,
+                        flow_job,
+                        is_simple,
+                        db,
+                        module,
+                        delete_after_use,
+                    )
+                    .await
                 }
-                LoopStatus::ParallelIteration { itered, .. } => {
+                ForLoopStatus::ParallelIteration { itered, .. } => {
                     let inner_path = Some(format!("{}/loop-parrallel", flow_job.script_path(),));
                     let continue_payload = if is_simple {
                         let payload = payload_from_simple_module(
@@ -2610,6 +2607,7 @@ async fn compute_next_flow_transform(
                                         cache_ttl: None,
                                         priority: None,
                                         early_return: None,
+                                        concurrency_key: None,
                                     },
                                     path: Some(format!("{}/forloop", flow_job.script_path())),
                                     restarted_from: None,
@@ -2708,6 +2706,7 @@ async fn compute_next_flow_transform(
                             cache_ttl: None,
                             priority: None,
                             early_return: None,
+                            concurrency_key: None,
                         },
                         path: Some(format!(
                             "{}/branchone-{}",
@@ -2759,6 +2758,7 @@ async fn compute_next_flow_transform(
                                                     cache_ttl: None,
                                                     priority: None,
                                                     early_return: None,
+                                                    concurrency_key: None,
                                                 },
                                                 path: Some(format!(
                                                     "{}/branchall-{}",
@@ -2826,6 +2826,7 @@ async fn compute_next_flow_transform(
                             cache_ttl: None,
                             priority: None,
                             early_return: None,
+                            concurrency_key: None,
                         },
                         path: Some(format!(
                             "{}/branchall-{}",
@@ -2844,6 +2845,220 @@ async fn compute_next_flow_transform(
     }
 }
 
+async fn next_loop_iteration(
+    flow: &FlowValue,
+    status: &FlowStatus,
+    ns: ForloopNextIteration,
+    modules: &Vec<FlowModule>,
+    flow_job: &QueuedJob,
+    is_simple: bool,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    module: &FlowModule,
+    delete_after_use: bool,
+) -> Result<NextFlowTransform, Error> {
+    let mut fm = flow.failure_module.clone();
+    if let Some(mut failure_module) = flow.failure_module.clone() {
+        failure_module.id_append(&format!("{}/{}", status.step, ns.index));
+        fm = Some(failure_module);
+    }
+    let mut modules = (*modules).clone();
+    add_virtual_items_if_necessary(&mut modules);
+    let inner_path = Some(format!("{}/loop-{}", flow_job.script_path(), ns.index));
+    if is_simple {
+        let payload =
+            payload_from_simple_module(&modules[0].value, db, flow_job, module, inner_path).await?;
+        Ok(NextFlowTransform::Continue(
+            ContinuePayload::SingleJob(payload),
+            NextStatus::NextLoopIteration {
+                next: ns,
+                simple_input_transforms: if is_simple {
+                    match &modules[0].value {
+                        FlowModuleValue::Script { input_transforms, .. }
+                        | FlowModuleValue::RawScript { input_transforms, .. }
+                        | FlowModuleValue::Flow { input_transforms, .. } => {
+                            Some(input_transforms.clone())
+                        }
+                        _ => None,
+                    }
+                } else {
+                    None
+                },
+            },
+        ))
+    } else {
+        Ok(NextFlowTransform::Continue(
+            ContinuePayload::SingleJob(JobPayloadWithTag {
+                payload: JobPayload::RawFlow {
+                    value: FlowValue {
+                        modules,
+                        failure_module: fm,
+                        same_worker: flow.same_worker,
+                        concurrent_limit: None,
+                        concurrency_time_window_s: None,
+                        skip_expr: None,
+                        cache_ttl: None,
+                        priority: None,
+                        early_return: None,
+                        concurrency_key: None,
+                    },
+                    path: inner_path,
+                    restarted_from: None,
+                },
+                tag: None,
+                delete_after_use,
+                timeout: None,
+            }),
+            NextStatus::NextLoopIteration { next: ns, simple_input_transforms: None },
+        ))
+    }
+}
+
+fn is_simple_modules(modules: &Vec<FlowModule>, flow: &FlowValue) -> bool {
+    let is_simple = modules.len() == 1
+        && modules[0].value.is_simple()
+        && modules[0].sleep.is_none()
+        && modules[0].suspend.is_none()
+        && modules[0].cache_ttl.is_none()
+        && (modules[0].mock.is_none()
+            && modules[0].mock.as_ref().is_some_and(|m| !m.enabled)
+            && flow.failure_module.is_none());
+    is_simple
+}
+
+async fn next_forloop_status(
+    status_module: &FlowStatusModule,
+    by_id: Option<IdContext>,
+    flow_job: &QueuedJob,
+    previous_id: &str,
+    status: &FlowStatus,
+    iterator: &InputTransform,
+    arc_last_job_result: Arc<Box<RawValue>>,
+    resumes: Arc<Box<RawValue>>,
+    resume: Arc<Box<RawValue>>,
+    approvers: Arc<Box<RawValue>>,
+    arc_flow_job_args: Arc<HashMap<String, Box<RawValue>>>,
+    client: &AuthedClient,
+    parallel: &bool,
+) -> Result<ForLoopStatus, Error> {
+    let next_loop_status = match status_module {
+        FlowStatusModule::WaitingForPriorSteps { .. }
+        | FlowStatusModule::WaitingForEvents { .. }
+        | FlowStatusModule::WaitingForExecutor { .. } => {
+            let by_id = if let Some(x) = by_id {
+                x
+            } else {
+                get_transform_context(&flow_job, previous_id, &status).await?
+            };
+            /* Iterator is an InputTransform, evaluate it into an array. */
+            let itered_raw = match iterator {
+                InputTransform::Static { value } => to_raw_value(value),
+                InputTransform::Javascript { expr } => {
+                    let mut context = HashMap::with_capacity(5);
+                    context.insert("result".to_string(), arc_last_job_result.clone());
+                    context.insert("previous_result".to_string(), arc_last_job_result);
+                    context.insert("resumes".to_string(), resumes);
+                    context.insert("resume".to_string(), resume);
+                    context.insert("approvers".to_string(), approvers);
+
+                    eval_timeout(
+                        expr.to_string(),
+                        context,
+                        Some(arc_flow_job_args),
+                        Some(client),
+                        Some(by_id),
+                    )
+                    .await?
+                }
+            };
+            let itered = serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
+                |not_array| {
+                    Error::ExecutionErr(format!(
+                        "Expected an array value in the iterator expression, found: {not_array}"
+                    ))
+                },
+            )?;
+
+            if itered.is_empty() {
+                ForLoopStatus::EmptyIterator
+            } else if *parallel {
+                ForLoopStatus::ParallelIteration { itered }
+            } else if let Some(first) = itered.first() {
+                let iter = Iter { index: 0 as i32, value: first.to_owned() };
+                ForLoopStatus::NextIteration(ForloopNextIteration {
+                    index: 0,
+                    itered,
+                    flow_jobs: vec![],
+                    new_args: iter,
+                    while_loop: false,
+                })
+            } else {
+                panic!("itered cannot be empty")
+            }
+        }
+
+        FlowStatusModule::InProgress {
+            iterator: Some(windmill_common::flow_status::Iterator { itered, index }),
+            flow_jobs: Some(flow_jobs),
+            ..
+        } if !*parallel => {
+            let itered_new = if itered.is_empty() {
+                // it's possible we need to re-compute the iterator Input Transforms here, in particular if the flow is being restarted inside the loop
+                let by_id = if let Some(x) = by_id {
+                    x
+                } else {
+                    get_transform_context(&flow_job, previous_id, &status).await?
+                };
+                let itered_raw = match iterator {
+                    InputTransform::Static { value } => to_raw_value(value),
+                    InputTransform::Javascript { expr } => {
+                        let mut context = HashMap::with_capacity(5);
+                        context.insert("result".to_string(), arc_last_job_result.clone());
+                        context.insert("previous_result".to_string(), arc_last_job_result);
+                        context.insert("resumes".to_string(), resumes);
+                        context.insert("resume".to_string(), resume);
+                        context.insert("approvers".to_string(), approvers);
+
+                        eval_timeout(
+                            expr.to_string(),
+                            context,
+                            Some(arc_flow_job_args),
+                            Some(client),
+                            Some(by_id),
+                        )
+                        .await?
+                    }
+                };
+                serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
+                    |not_array| {
+                        Error::ExecutionErr(format!("Expected an array value, found: {not_array}"))
+                    },
+                )?
+            } else {
+                itered.clone()
+            };
+            let (index, next) = index
+                .checked_add(1)
+                .and_then(|i| itered_new.get(i).map(|next| (i, next)))
+                .with_context(|| {
+                    format!("Could not find iteration number {index} restarting inside the for-loop flow. It's possible the itered-array has changed and this value isn't available anymore.")
+                })?;
+
+            ForLoopStatus::NextIteration(ForloopNextIteration {
+                index,
+                itered: itered_new.clone(),
+                flow_jobs: flow_jobs.clone(),
+                new_args: Iter { index: index as i32, value: next.to_owned() },
+                while_loop: false,
+            })
+        }
+
+        _ => Err(Error::BadRequest(format!(
+            "Unrecognized module status for ForloopFlow {status_module:?}"
+        )))?,
+    };
+    Ok(next_loop_status)
+}
+
 async fn payload_from_simple_module(
     value: &FlowModuleValue,
     db: &sqlx::Pool<sqlx::Postgres>,