all

* chore(main): release 1.224.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/DockerfileBackendTests

@@ -41,7 +41,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 
-RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 

.github/workflows/backend-test.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   cargo_test:
-    runs-on: [self-hosted, new]
+    runs-on: ubicloud-standard-8
     container:
       image: ghcr.io/windmill-labs/backend-tests
     services:
@@ -31,15 +31,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: |
-            backend
-            backend -> target
+      # - uses: Swatinem/rust-cache@v2
+      #   with:
+      #     workspaces: |
+      #       backend
+      #       backend -> target
       - name: cargo test
-        timeout-minutes: 10
+        timeout-minutes: 15
         run:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
-          DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_EMBEDDING=true cargo
-          test --features enterprise --all -- --nocapture 
+          DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill
+          DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features enterprise --all --
+          --nocapture

.github/workflows/benchmark.yml

@@ -7,14 +7,13 @@ on:
 
 jobs:
   benchmark:
-    runs-on: [self-hosted, new]
+    runs-on: ubicloud-standard-8
     services:
       postgres:
         image: postgres
         env:
           POSTGRES_DB: windmill
           POSTGRES_PASSWORD: changeme
-
         options: >-
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
@@ -35,12 +34,12 @@ jobs:
         image: ghcr.io/windmill-labs/windmill-ee:main
         env:
           DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
-          DISABLE_SERVER: true
+          MODE: worker
           WORKER_GROUP: dedicated
           DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
           LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
         options: >-
-          --pull always
+          --pull always --restart unless-stopped
     steps:
       - uses: denoland/setup-deno@v1
         with:
@@ -50,9 +49,11 @@ jobs:
           ref: benchmarks
       - name: benchmark
         timeout-minutes: 20
-        run: deno run --unstable -A -r
+        run:
+          deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
       - name: Push changes
         run: |
           pwd

.github/workflows/change-versions.yml

@@ -6,7 +6,7 @@ on:
       - "version.txt"
 jobs:
   change_version:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     container: node:18
     steps:
       - uses: actions/checkout@v3

.github/workflows/clean-docker.yml

@@ -1,13 +0,0 @@
-name: Clean docker
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 */2 * *"
-
-jobs:
-  build:
-    runs-on: [self-hosted, new]
-    steps:
-      - name: clean docker
-        run: |
-          sudo docker system prune -f

.github/workflows/deno_on_release.yml

@@ -8,7 +8,7 @@ env:
 
 jobs:
   build_deno_and_push_to_repo:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     steps:
       - uses: actions/checkout@v3
       - name: generate_deno
@@ -31,7 +31,7 @@ jobs:
 
   tag_repo:
     needs: [build_deno_and_push_to_repo]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/docker-310.yml.archived

@@ -16,7 +16,7 @@ permissions:
 
 jobs:
   build_ee:
-    runs-on: ubuntu-22.04
+    runs-on: ubicloud
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/docker-image.yml

@@ -20,7 +20,7 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     steps:
       - uses: actions/checkout@v3
         with:
@@ -63,7 +63,7 @@ jobs:
             org.opencontainers.image.licenses=AGPLv3
 
   build_ee:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     steps:
       - uses: actions/checkout@v3
         with:
@@ -99,7 +99,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise
+            features=enterprise,saml
             nsjail=true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
@@ -108,6 +108,52 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
+  build_ee_reports_privately:
+    needs: [build_ee]
+    runs-on: ubicloud
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+
+      - uses: depot/setup-action@v1
+
+      - name: Docker meta
+        id: meta-ee-public
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=short
+
+      - name: Login to ECR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          file: "./docker/DockerfileReports"
+          tags: |
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License
+
   # disabled until we make it 100% reliable and add more meaningful tests
   # playwright:
   #   runs-on: [self-hosted, new]
@@ -143,7 +189,7 @@ jobs:
 
   deploy_s3:
     needs: [build_ee]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     if: github.event_name != 'pull_request'
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -165,10 +211,9 @@ jobs:
           bucket: windmill-frontend
           bucket-region: us-east-1
 
-
   attach_amd64_binary_to_release:
     needs: [build, build_ee]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
     env:
       ARCH: amd64
@@ -203,7 +248,6 @@ jobs:
           mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
           mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
 
-
       - name: Attach binary to release
         uses: softprops/action-gh-release@v1
         with:
@@ -213,7 +257,7 @@ jobs:
 
   attach_arm64_binary_to_release:
     needs: [build, build_ee]
-    runs-on: ubuntu-latest
+    runs-on: ubicoud
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
     env:
       ARCH: arm64
@@ -248,7 +292,6 @@ jobs:
           mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
           mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
 
-
       - name: Attach binary to release
         uses: softprops/action-gh-release@v1
         with:
@@ -258,7 +301,7 @@ jobs:
 
   publish_ecr:
     needs: [build_ee]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud
     if: github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@v3

.github/workflows/docker-mssql.yml.archived

@@ -5,8 +5,6 @@ env:
 name: Build windmill:mssql
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "0 0 */4 * *"
 
 concurrency:
   group: ${{ github.ref }}-mssql

.github/workflows/frontend-check.yml

@@ -1,14 +1,14 @@
 name: check frontend build
 on:
   pull_request:
-    types: [opened,synchronize,reopened,closed]
+    types: [opened, synchronize, reopened, closed]
     paths:
       - "frontend/**"
   merge_group:
 
 jobs:
   npm_check:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -16,4 +16,6 @@ jobs:
           node-version: 18
       - name: "npm check"
         timeout-minutes: 2
-        run: cd frontend && npm ci && npm run generate-backend-client && npm run check
+        run:
+          cd frontend && npm ci && npm run generate-backend-client && npm run
+          check

.github/workflows/go_on_release.yml

@@ -10,7 +10,7 @@ env:
 
 jobs:
   build_go_and_push_to_repo:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4

.github/workflows/npm_on_release.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build_npm:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

.github/workflows/publish_lsp.yml

@@ -16,14 +16,17 @@ permissions:
   packages: write
 
 jobs:
-  publish_lsp:
-    runs-on: ubuntu-latest
+  sleep:
+    runs-on: ubicloud
     steps:
       - name: Sleep for 900 seconds waiting for pypi to update index
         if: startsWith(github.ref, 'refs/tags/v')
         run: sleep 900
         shell: bash
-
+  publish_lsp:
+    needs: [sleep]
+    runs-on: ubicloud
+    steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -62,7 +65,7 @@ jobs:
 
   publish_lsp_private:
     needs: [publish_lsp]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/pypi_on_release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   publish_pypi:
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     if: startsWith(github.ref, 'refs/tags/v')
     container:
       image: ghcr.io/windmill-labs/python-client-builder

.github/workflows/release-please.yml

@@ -6,7 +6,7 @@ name: release-please
 jobs:
   release-please:
     name: "Release please"
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: GoogleCloudPlatform/release-please-action@v3
         with:

CHANGELOG.md

@@ -1,6 +1,303 @@
 # Changelog
 
 
+## [1.224.0](https://github.com/windmill-labs/windmill/compare/v1.223.1...v1.224.0) (2023-12-10)
+
+
+### Features
+
+* add authelia sso support ([#2824](https://github.com/windmill-labs/windmill/issues/2824)) ([6fefdb5](https://github.com/windmill-labs/windmill/commit/6fefdb5438d0fd7546759d93b5f4465731a4cac5))
+* embedded server only get killed after last job of worker ([4f4ca26](https://github.com/windmill-labs/windmill/commit/4f4ca264aef3e1d3ee4748593f0ebcb425d333ef))
+
+
+### Bug Fixes
+
+* make cleanup_module backcompatible ([58c4c0b](https://github.com/windmill-labs/windmill/commit/58c4c0b5c2b009ff56e3eb6bd19e766dbff4cf54))
+
+## [1.223.1](https://github.com/windmill-labs/windmill/compare/v1.223.0...v1.223.1) (2023-12-08)
+
+
+### Bug Fixes
+
+* improve aggrid selectedRow output + make first selectedRow configurable ([651d998](https://github.com/windmill-labs/windmill/commit/651d99824d1932a48fdb9ffd5dff944fa2c2c8aa))
+
+## [1.223.0](https://github.com/windmill-labs/windmill/compare/v1.222.0...v1.223.0) (2023-12-08)
+
+
+### Features
+
+* Add a git sync test button ([#2817](https://github.com/windmill-labs/windmill/issues/2817)) ([535202c](https://github.com/windmill-labs/windmill/commit/535202c1fe1e3a90b1aff2dad6632f615ca8aef8))
+* add mysql datetime ([#2808](https://github.com/windmill-labs/windmill/issues/2808)) ([8896a8c](https://github.com/windmill-labs/windmill/commit/8896a8cacffb5da5575203a5fc9aa6863b303fde))
+* Add possibility to delete flow step results when the flow is complete ([#2806](https://github.com/windmill-labs/windmill/issues/2806)) ([b65657d](https://github.com/windmill-labs/windmill/commit/b65657d0f8db21f87a90bdcd6ce1122a8bb209ce))
+* no worker with tag warning ([#2803](https://github.com/windmill-labs/windmill/issues/2803)) ([e7141c1](https://github.com/windmill-labs/windmill/commit/e7141c147d9ff4728399ae0703ca636c776cb264))
+* sql types autocomplete ([#2810](https://github.com/windmill-labs/windmill/issues/2810)) ([a9b5d6b](https://github.com/windmill-labs/windmill/commit/a9b5d6bebb00c96cc329b4178816c73db5cbe3f0))
+
+
+### Bug Fixes
+
+* conditional component also for toggles ([d26730a](https://github.com/windmill-labs/windmill/commit/d26730ad3419df5c5d75328e7f441c701b2c0c68))
+* db schema autocomplete ([#2809](https://github.com/windmill-labs/windmill/issues/2809)) ([e250897](https://github.com/windmill-labs/windmill/commit/e2508972378f455b7af7984f2c33894dadd4f7bc))
+* deno chromium support when nsjail enabled ([#2815](https://github.com/windmill-labs/windmill/issues/2815)) ([2964569](https://github.com/windmill-labs/windmill/commit/296456996293d861c7cab2d10e01e3b6443e421c))
+* Fix test_complex_flow_restart ([#2813](https://github.com/windmill-labs/windmill/issues/2813)) ([b089449](https://github.com/windmill-labs/windmill/commit/b0894492f1e3c06880c0e1875c8c8886e31a165f))
+* improve sql default arg parsing + auto invite ([033000f](https://github.com/windmill-labs/windmill/commit/033000fed36d6172cf3b9a83ff449f5ba051fa08))
+* support interval in pg ([009a83d](https://github.com/windmill-labs/windmill/commit/009a83dd9bbc115b94978d2cb3113cbe10acd05d))
+
+## [1.222.0](https://github.com/windmill-labs/windmill/compare/v1.221.0...v1.222.0) (2023-12-07)
+
+
+### Features
+
+* conditional fields on forms ([b10df30](https://github.com/windmill-labs/windmill/commit/b10df302455202940348cd20c2e733734fb7d027))
+* mysql named params ([#2805](https://github.com/windmill-labs/windmill/issues/2805)) ([c4a1054](https://github.com/windmill-labs/windmill/commit/c4a1054a5a4ab3fd700de842a1f46d5886632b11))
+
+
+### Bug Fixes
+
+* add saml feature flag ([#2802](https://github.com/windmill-labs/windmill/issues/2802)) ([da15a56](https://github.com/windmill-labs/windmill/commit/da15a56a07288d4164c88289dd559389735db719))
+* app reports screenshot refresh ([#2800](https://github.com/windmill-labs/windmill/issues/2800)) ([f7f7130](https://github.com/windmill-labs/windmill/commit/f7f71302f7d0fcac7468d506ebaceb55332ec1ed))
+* fix extraOptions for agGrid ([f7d7746](https://github.com/windmill-labs/windmill/commit/f7d7746327411da06cede4e0ff06222d318d01bc))
+* fix extraOptions for agGrid ([2f7fb9b](https://github.com/windmill-labs/windmill/commit/2f7fb9bfd490932a2f0bc01f02df678668487f8c))
+* **frontend:** Fix chartjs options merge ([#2804](https://github.com/windmill-labs/windmill/issues/2804)) ([e11b257](https://github.com/windmill-labs/windmill/commit/e11b257b2d761b2b26160d61f8e113dbe47e9efe))
+
+## [1.221.0](https://github.com/windmill-labs/windmill/compare/v1.220.0...v1.221.0) (2023-12-06)
+
+
+### Features
+
+* limit pro plans ([#2794](https://github.com/windmill-labs/windmill/issues/2794)) ([c58190e](https://github.com/windmill-labs/windmill/commit/c58190ef788feaae9bac9848e8cfa59d10f51cab))
+
+
+### Bug Fixes
+
+* Allow run git for Deno ([#2789](https://github.com/windmill-labs/windmill/issues/2789)) ([8ad8d20](https://github.com/windmill-labs/windmill/commit/8ad8d20136f3151ca28784ba3513af562fcb5c9b))
+* Allow run git for Deno ([#2790](https://github.com/windmill-labs/windmill/issues/2790)) ([d822beb](https://github.com/windmill-labs/windmill/commit/d822beba63f08e4e1e1dac4aff56353276558f02))
+* app reports nits ([#2788](https://github.com/windmill-labs/windmill/issues/2788)) ([0b13f2d](https://github.com/windmill-labs/windmill/commit/0b13f2d782d5626d64b5efe39232ec8da268cd27))
+* app reports ts ([#2798](https://github.com/windmill-labs/windmill/issues/2798)) ([4f6a116](https://github.com/windmill-labs/windmill/commit/4f6a116e462d198cb81260749fdd8d57bc0ef03a))
+* Deno allow run git ([#2791](https://github.com/windmill-labs/windmill/issues/2791)) ([256ee64](https://github.com/windmill-labs/windmill/commit/256ee646b49bca091ee6e7a0fea37f7d98c9e378))
+* **frontend:** fix storage key typo ([#2796](https://github.com/windmill-labs/windmill/issues/2796)) ([e4da819](https://github.com/windmill-labs/windmill/commit/e4da819cda12e0e1f27cabc594fe5af5e2d9966c))
+* **frontend:** use correct id for selected for loop iteration ([#2787](https://github.com/windmill-labs/windmill/issues/2787)) ([90668fb](https://github.com/windmill-labs/windmill/commit/90668fb0e5b4ac9f09a0b1e57fcb10b4b4208d2e))
+* improve conditionalwrapper selection ([b5f5355](https://github.com/windmill-labs/windmill/commit/b5f53556a9afb3becbaaab034eeb8fdf4b1178a1))
+* linked variable already being a variable ([9c4f1e0](https://github.com/windmill-labs/windmill/commit/9c4f1e0333743b35c237b4d8fe49757c8ae122e7))
+* smtp doesn't require username/password ([abb50fa](https://github.com/windmill-labs/windmill/commit/abb50fac93a615d0ee18e9776c1fb84829fa2464))
+
+## [1.220.0](https://github.com/windmill-labs/windmill/compare/v1.219.1...v1.220.0) (2023-12-05)
+
+
+### Features
+
+* Add workspace settings to sync scripts/flows/apps to git repo on deployment ([#2740](https://github.com/windmill-labs/windmill/issues/2740)) ([194ee83](https://github.com/windmill-labs/windmill/commit/194ee830cc8d8cc0ea69c877b428966f3f20bdb5))
+* app reports v2 ([#2778](https://github.com/windmill-labs/windmill/issues/2778)) ([f36a510](https://github.com/windmill-labs/windmill/commit/f36a510025d87fc20e15f66e041390cc701c65c8))
+* CLI now accepts message arg for sync push ([#2784](https://github.com/windmill-labs/windmill/issues/2784)) ([d9458e9](https://github.com/windmill-labs/windmill/commit/d9458e96ebc26160e9d5f909574bd1cf1d288d73))
+* Custom content type for script and flow results ([#2767](https://github.com/windmill-labs/windmill/issues/2767)) ([6c0f921](https://github.com/windmill-labs/windmill/commit/6c0f921b3ab3acac8890ceb1fa0565bf082de0d4))
+* **frontend:** add before and after icons for text input components ([#2746](https://github.com/windmill-labs/windmill/issues/2746)) ([a3f1111](https://github.com/windmill-labs/windmill/commit/a3f1111ca7715ffaa3a615786adee20e00870b30))
+* History commit message can be updated from the UI ([#2777](https://github.com/windmill-labs/windmill/issues/2777)) ([874b7a5](https://github.com/windmill-labs/windmill/commit/874b7a50f72a6505490a02c82786ea5f5c8261e0))
+* perpetual scripts ([bea0da3](https://github.com/windmill-labs/windmill/commit/bea0da3d6ff81ace956d6a1e7fb643e449927b05))
+* result preview in app editor ([#2761](https://github.com/windmill-labs/windmill/issues/2761)) ([51d648d](https://github.com/windmill-labs/windmill/commit/51d648d450c578580f6d9a9630ff91920060cc27))
+* save inline scripts of apps to workspace + search by path ([f50cef0](https://github.com/windmill-labs/windmill/commit/f50cef03670e50b8d7a6bb4719ae636f2714f22f))
+* Various minor improvements for S3 DX ([#2760](https://github.com/windmill-labs/windmill/issues/2760)) ([5cb3c34](https://github.com/windmill-labs/windmill/commit/5cb3c34e440be878687217fdd95d735f053c7839))
+
+
+### Bug Fixes
+
+* add access-control-allow-origin to static assets ([60cd14f](https://github.com/windmill-labs/windmill/commit/60cd14ff59a80f671838a5f9bfa21effc309cb54))
+* add run_flow_sync to python client + few nits ([042504f](https://github.com/windmill-labs/windmill/commit/042504f9052963c882ac346d2026c518c6d94f0f))
+* Array of S3 files now shows links to the S3 explorer ([#2783](https://github.com/windmill-labs/windmill/issues/2783)) ([20b0500](https://github.com/windmill-labs/windmill/commit/20b0500a2824be1e715d4301e05c6081a5575dbe))
+* deployment_metadata table app_version is a nullable BIGINT ([#2769](https://github.com/windmill-labs/windmill/issues/2769)) ([df86538](https://github.com/windmill-labs/windmill/commit/df865380bceb8663defb322a88f58f7c3f1a263c))
+* do not prorate for team plan checkouts mornings of firsts ([#2751](https://github.com/windmill-labs/windmill/issues/2751)) ([8fc0afc](https://github.com/windmill-labs/windmill/commit/8fc0afce714774647f86e1aca8eeec4b7d4c13fc))
+* fix early return sync on existing flows ([60c4860](https://github.com/windmill-labs/windmill/commit/60c486023310ef85d1cb171ebca792be8d7917e8))
+* **frontend:** Fix dark mode observer, removed code duplication ([#2770](https://github.com/windmill-labs/windmill/issues/2770)) ([a6d1e7c](https://github.com/windmill-labs/windmill/commit/a6d1e7c79bb16d2c02d53f9ede56451e064956e0))
+* **frontend:** fix fileinput drag and drop check ([#2781](https://github.com/windmill-labs/windmill/issues/2781)) ([0c3d4a6](https://github.com/windmill-labs/windmill/commit/0c3d4a64c4a8ad823646f9ce27de7e58ccc002e8))
+* **frontend:** fix resource search ([#2748](https://github.com/windmill-labs/windmill/issues/2748)) ([6eaec47](https://github.com/windmill-labs/windmill/commit/6eaec471629d6c60e6c3123372fa3569bc5471c7))
+* **frontend:** fix sign in with Google button ([#2756](https://github.com/windmill-labs/windmill/issues/2756)) ([8ca04e4](https://github.com/windmill-labs/windmill/commit/8ca04e49b061675f820e4aeaad6196403eabe135))
+* **frontend:** fix tutorials contols ([#2745](https://github.com/windmill-labs/windmill/issues/2745)) ([85805bd](https://github.com/windmill-labs/windmill/commit/85805bdf8265f4fe14a583b43b786257ffccc4dc))
+* **frontend:** infer args if the schema is not an object ([#2782](https://github.com/windmill-labs/windmill/issues/2782)) ([b67d005](https://github.com/windmill-labs/windmill/commit/b67d00540b7adfac3550299d50fc9a620b5a6e2e))
+* improve azure openai readme ([#2754](https://github.com/windmill-labs/windmill/issues/2754)) ([d47d4cc](https://github.com/windmill-labs/windmill/commit/d47d4ccf85747a14e0498be871ead08371392ff5))
+* show bg runnables currently running ([a122b6c](https://github.com/windmill-labs/windmill/commit/a122b6cab664e0b8b3d7223b9fd70569a2fe0991))
+* string or enum ([#2752](https://github.com/windmill-labs/windmill/issues/2752)) ([583dae6](https://github.com/windmill-labs/windmill/commit/583dae6a72ee6be70dee5a356e1b4172db9a7e2d))
+* string results with custom content types are not quoted ([#2768](https://github.com/windmill-labs/windmill/issues/2768)) ([1cad51b](https://github.com/windmill-labs/windmill/commit/1cad51b5941a3bd1892c935581373bb7375da45a))
+* strip prefix of s3 resource ([#2780](https://github.com/windmill-labs/windmill/issues/2780)) ([f3a0c81](https://github.com/windmill-labs/windmill/commit/f3a0c81639662a7da65af49e4032dfc573dd93eb))
+
+## [1.219.1](https://github.com/windmill-labs/windmill/compare/v1.219.0...v1.219.1) (2023-12-01)
+
+
+### Bug Fixes
+
+* fix editorContext in preview ([cb42f10](https://github.com/windmill-labs/windmill/commit/cb42f10dc605f00ba70fadc61069aba98af8ec88))
+* maps mapRegion update on move ([62de305](https://github.com/windmill-labs/windmill/commit/62de305908ba7e6ac75aa5e8ffba822522b08345))
+
+## [1.219.0](https://github.com/windmill-labs/windmill/compare/v1.218.0...v1.219.0) (2023-12-01)
+
+
+### Features
+
+* eval preview for apps ([d4374a0](https://github.com/windmill-labs/windmill/commit/d4374a0103d2244b31b5cec7649dbfb96b2af1b4))
+
+
+### Bug Fixes
+
+* **frontend:** rename menu component ([#2738](https://github.com/windmill-labs/windmill/issues/2738)) ([f816ad0](https://github.com/windmill-labs/windmill/commit/f816ad01d3e39917714abadcf3833a4d8619e4f3))
+* improve quickstyle to be in static ccomponents only ([4774e03](https://github.com/windmill-labs/windmill/commit/4774e03be5663017bccbe95f026cc39d4c43f536))
+* improve stat card ([fc5f054](https://github.com/windmill-labs/windmill/commit/fc5f054b94fa58ee889d10d06e486b6e8c4f885e))
+* limit log pull from queued jobs ([592d783](https://github.com/windmill-labs/windmill/commit/592d7839d183843b425521a319b802fb8cac3f21))
+* support INET in pg ([007d5be](https://github.com/windmill-labs/windmill/commit/007d5be23e038fbf82907c3b335b3d747791295d))
+
+## [1.218.0](https://github.com/windmill-labs/windmill/compare/v1.217.0...v1.218.0) (2023-11-30)
+
+
+### Features
+
+* **frontend:** add menu component ([#2721](https://github.com/windmill-labs/windmill/issues/2721)) ([0665692](https://github.com/windmill-labs/windmill/commit/06656924ae8173e906a243aab2af658a1689af01))
+* **frontend:** resource rework ([#2725](https://github.com/windmill-labs/windmill/issues/2725)) ([ec25856](https://github.com/windmill-labs/windmill/commit/ec25856b7c8a7d33a50d1beb5f3e99c7b912e1ca))
+
+
+### Bug Fixes
+
+* make REST language support URLSearchParams, headers, FormData ([830dec0](https://github.com/windmill-labs/windmill/commit/830dec0f90e0189b4dea4de8c44a03e437acf440))
+* Pull patched version of gosyn ([#2734](https://github.com/windmill-labs/windmill/issues/2734)) ([0bf3685](https://github.com/windmill-labs/windmill/commit/0bf3685cbdecae0a8e7a24a5198ae2fed98fe340))
+* s3 resource openapi spec ([#2730](https://github.com/windmill-labs/windmill/issues/2730)) ([16d10ae](https://github.com/windmill-labs/windmill/commit/16d10aedf79cddf7cdc15d55c42c72f4948b3ee2))
+* update deployed even when draft only ([#2694](https://github.com/windmill-labs/windmill/issues/2694)) ([1f99fcd](https://github.com/windmill-labs/windmill/commit/1f99fcd3af21208676aa90015559359740b0534a))
+* Using latest gosyn ([#2737](https://github.com/windmill-labs/windmill/issues/2737)) ([920cc9a](https://github.com/windmill-labs/windmill/commit/920cc9a576db0112ffd9572480c3d0a8aa08055b))
+* Workspace error handler creation was not adding the slack resource ([#2733](https://github.com/windmill-labs/windmill/issues/2733)) ([70c504e](https://github.com/windmill-labs/windmill/commit/70c504edfaf7c114c91af2b3ca39dfee073f04f2))
+
+## [1.217.0](https://github.com/windmill-labs/windmill/compare/v1.216.0...v1.217.0) (2023-11-29)
+
+
+### Features
+
+* add support for raw query args ([8275602](https://github.com/windmill-labs/windmill/commit/82756023728fe392fee93a7bba0567ce582aad8c))
+* **frontend:** add "hide schedules" filter ([#2710](https://github.com/windmill-labs/windmill/issues/2710)) ([46e0f91](https://github.com/windmill-labs/windmill/commit/46e0f913878c9f688594cfe1c9184438e4facf49))
+* **frontend:** Added tailwind classes auto-complete ([#2712](https://github.com/windmill-labs/windmill/issues/2712)) ([2d3ce8a](https://github.com/windmill-labs/windmill/commit/2d3ce8a49c952accb4f1ceb0e6ea608df539f179))
+* **frontend:** Stat card improvement ([#2709](https://github.com/windmill-labs/windmill/issues/2709)) ([89abb68](https://github.com/windmill-labs/windmill/commit/89abb68f635d062778485c15d88fd6272ebd40a2))
+* scheduled app reports ([#2714](https://github.com/windmill-labs/windmill/issues/2714)) ([3789b34](https://github.com/windmill-labs/windmill/commit/3789b34dae72ff57bb9e0bb7e93439446c78095d))
+
+
+### Bug Fixes
+
+* ai fix popup placement + update edit/fix prompt to return complete code ([#2715](https://github.com/windmill-labs/windmill/issues/2715)) ([76a387f](https://github.com/windmill-labs/windmill/commit/76a387f4a181d21afad0138c07f947aa292978cc))
+* better error for moved openai resource ([#2724](https://github.com/windmill-labs/windmill/issues/2724)) ([15b2c9f](https://github.com/windmill-labs/windmill/commit/15b2c9f171122ab074f8430a785f8ed577921fcc))
+* Deno can talk to private NPM registries behind HTTPS ([#2713](https://github.com/windmill-labs/windmill/issues/2713)) ([726866b](https://github.com/windmill-labs/windmill/commit/726866b410863ca9583a8145ebd9f4c4557cef08))
+* Error handler now supports flows ([#2707](https://github.com/windmill-labs/windmill/issues/2707)) ([36e46e2](https://github.com/windmill-labs/windmill/commit/36e46e2e47e1e949e1e235fa4ec0365b46d80de1))
+* **frontend:** fix separator z-index ([#2720](https://github.com/windmill-labs/windmill/issues/2720)) ([9525ab7](https://github.com/windmill-labs/windmill/commit/9525ab7bbab9f820d1f6fee44f9a2e9ad19d1b54))
+* **frontend:** Fix table initial ordering ([#2727](https://github.com/windmill-labs/windmill/issues/2727)) ([1a1d1db](https://github.com/windmill-labs/windmill/commit/1a1d1db96fd069434291f4b5f37ae1d0e21c5e44))
+* **frontend:** Improve error message + fix overflow when file name is too long ([#2691](https://github.com/windmill-labs/windmill/issues/2691)) ([c990f85](https://github.com/windmill-labs/windmill/commit/c990f856aaf821899cc54db20a653af396ceae7f))
+* generate cargo lock file ([#2722](https://github.com/windmill-labs/windmill/issues/2722)) ([bd31979](https://github.com/windmill-labs/windmill/commit/bd31979a62a30071e68385b5eb5dc28f37ea8fb5))
+* improve autocomplete reactivity ([c3eaf0b](https://github.com/windmill-labs/windmill/commit/c3eaf0bf4aab531b52a1b3ac276c6840b0925786))
+* make dedicated workers able to redeploy automatically ([e6d67f4](https://github.com/windmill-labs/windmill/commit/e6d67f4e5994360e7ae83c1303b31514b6062d1d))
+* minor fixes to private NPM and python registries to get everything working ([#2728](https://github.com/windmill-labs/windmill/issues/2728)) ([3d6fb15](https://github.com/windmill-labs/windmill/commit/3d6fb15a90f4d5aa18a6d6158760a380295e3d9e))
+* only list session and permanent token in user settings ([c8046af](https://github.com/windmill-labs/windmill/commit/c8046af9d01ade6891ea97e56974bb742bbf3e6e))
+* trim .bun.ts for local imports ([d2b3026](https://github.com/windmill-labs/windmill/commit/d2b3026032d288e1e5de1f37979d92d02094b3e4))
+
+## [1.216.0](https://github.com/windmill-labs/windmill/compare/v1.215.0...v1.216.0) (2023-11-26)
+
+
+### Features
+
+* add early return for flows ([dbaef0a](https://github.com/windmill-labs/windmill/commit/dbaef0aa5f22c55b691f6205ae053c155a0e025c))
+* fill pg resource from string ([#2699](https://github.com/windmill-labs/windmill/issues/2699)) ([f1cabb4](https://github.com/windmill-labs/windmill/commit/f1cabb40f6cbec05b105563dad58da048d9187ec))
+* **frontend:** add currency format + add enum autocomplete + fix run… ([#2670](https://github.com/windmill-labs/windmill/issues/2670)) ([0c0f43d](https://github.com/windmill-labs/windmill/commit/0c0f43dd3ac0541d889046c0c6df8dbf78650b9c))
+* **frontend:** add stat card ([#2687](https://github.com/windmill-labs/windmill/issues/2687)) ([81ef24b](https://github.com/windmill-labs/windmill/commit/81ef24b3cec18f86e407b343b4d72ac566d268e1))
+* **python:** Update return type for 'get_resource' function ([#2695](https://github.com/windmill-labs/windmill/issues/2695)) ([603e7ff](https://github.com/windmill-labs/windmill/commit/603e7ff67f5f68d291db37173b89325d5de911a1))
+
+
+### Bug Fixes
+
+* expand enum type narrowing to forms ([94e9b80](https://github.com/windmill-labs/windmill/commit/94e9b80e1993a217415118bf48b8575cd8363746))
+* **frontend:** correctly handle when result is undefined ([#2693](https://github.com/windmill-labs/windmill/issues/2693)) ([f4aa768](https://github.com/windmill-labs/windmill/commit/f4aa76897ea0f261c1e47976f517058601c479f9))
+* improve multiselect from form ([5fa653d](https://github.com/windmill-labs/windmill/commit/5fa653d154c6e8697119796a025bdc380f68de9a))
+* improve resource pages ([16be230](https://github.com/windmill-labs/windmill/commit/16be2300ff66cad8b740856a6435d1c1e53bce99))
+* lighten monaco editor workers ([#2690](https://github.com/windmill-labs/windmill/issues/2690)) ([4c42836](https://github.com/windmill-labs/windmill/commit/4c42836cfcad63da023a658213f03b45c925efb2))
+* Priority tags FE buggy when missing from config ([#2702](https://github.com/windmill-labs/windmill/issues/2702)) ([e8b1f22](https://github.com/windmill-labs/windmill/commit/e8b1f220dd299922a32b04451756f2b7fa735d5d))
+* S3 bucket browser small improvements and fixes ([#2700](https://github.com/windmill-labs/windmill/issues/2700)) ([55e34d8](https://github.com/windmill-labs/windmill/commit/55e34d8cdd64362615f2c1bc2698c0538c287784))
+
+## [1.215.0](https://github.com/windmill-labs/windmill/compare/v1.214.1...v1.215.0) (2023-11-23)
+
+
+### Features
+
+* query embeddings from s3 ([#2683](https://github.com/windmill-labs/windmill/issues/2683)) ([e3f2b43](https://github.com/windmill-labs/windmill/commit/e3f2b43748e8e0853dd0dbae87cba37b950dc76e))
+
+
+### Bug Fixes
+
+* bigquery schema and date inputs ([#2688](https://github.com/windmill-labs/windmill/issues/2688)) ([f5e098d](https://github.com/windmill-labs/windmill/commit/f5e098d03b467002a8deab4d2c519945c01458d4))
+* improve wmill go client ([cfd3da4](https://github.com/windmill-labs/windmill/commit/cfd3da41efe4e89e5d8672c08433442b05d11f37))
+
+## [1.214.1](https://github.com/windmill-labs/windmill/compare/v1.214.0...v1.214.1) (2023-11-23)
+
+
+### Features
+
+* **python:** Quality of Life improvements to Python client ([#2686](https://github.com/windmill-labs/windmill/issues/2686)) ([1c2abcd](https://github.com/windmill-labs/windmill/commit/1c2abcda231b2d10e7a358a4e4b7973785cb6199))
+
+
+### Bug Fixes
+
+* add image base64 source kinds ([7ae84fc](https://github.com/windmill-labs/windmill/commit/7ae84fce5014d97bf17803e2ccefafee009b28bb))
+* fix backend build ([45ee1d7](https://github.com/windmill-labs/windmill/commit/45ee1d770345f03c03cc48a3ddcc130ca3655a4c))
+* **frontend:** disable active interaction to avoid broken state ([#2675](https://github.com/windmill-labs/windmill/issues/2675)) ([bfdb559](https://github.com/windmill-labs/windmill/commit/bfdb559b47786e37f135ca345c55828cc70e49e9))
+* **frontend:** improve tutorial ux ([#2677](https://github.com/windmill-labs/windmill/issues/2677)) ([7fe3bca](https://github.com/windmill-labs/windmill/commit/7fe3bca624fa3b434e3fbe9b33b213449044accc))
+* **frontend:** use popover for schedule for later on the runs page ([#2678](https://github.com/windmill-labs/windmill/issues/2678)) ([31fbc58](https://github.com/windmill-labs/windmill/commit/31fbc5867cc72befea5733811148126289ce8bb9))
+* graphql variables ([#2682](https://github.com/windmill-labs/windmill/issues/2682)) ([217e694](https://github.com/windmill-labs/windmill/commit/217e69498fa5958e83c595003b2a086528e51388))
+* relax tags constraints ([8f8ea22](https://github.com/windmill-labs/windmill/commit/8f8ea227c850f9d01f913660547b6a6394c6e142))
+* Various fixes and improvements for Windmill S3 capabilities ([#2674](https://github.com/windmill-labs/windmill/issues/2674)) ([c42c54e](https://github.com/windmill-labs/windmill/commit/c42c54e69b5acdcbe44a0c26ae7fbaa2189e6b5f))
+
+## [1.214.0](https://github.com/windmill-labs/windmill/compare/v1.213.0...v1.214.0) (2023-11-22)
+
+
+### Features
+
+* hub path scripts + nested inputs glue ([#2668](https://github.com/windmill-labs/windmill/issues/2668)) ([ad199af](https://github.com/windmill-labs/windmill/commit/ad199afd06814540bb7d36669709902be56eeb8a))
+* **python:** Refactor Windmill Python client (remove `windmill-api`) ([#2665](https://github.com/windmill-labs/windmill/issues/2665)) ([37ffdae](https://github.com/windmill-labs/windmill/commit/37ffdaed60fb750f1466b440353e1d8409eaea90))
+
+
+### Bug Fixes
+
+* fix flow primary schedule clearing ([3ebe6d7](https://github.com/windmill-labs/windmill/commit/3ebe6d7a620e37fd6c81bcf4c8713a48eb081f81))
+
+## [1.213.0](https://github.com/windmill-labs/windmill/compare/v1.212.0...v1.213.0) (2023-11-21)
+
+
+### Features
+
+* code completion UI + other nits ([#2657](https://github.com/windmill-labs/windmill/issues/2657)) ([6d426b4](https://github.com/windmill-labs/windmill/commit/6d426b4ec49d4749a89c86dc5eb1f11bf705ca26))
+* Expanding an s3object result now opens the S3 file browser ([#2656](https://github.com/windmill-labs/windmill/issues/2656)) ([baac93f](https://github.com/windmill-labs/windmill/commit/baac93f40140ee37548a273885c028a8e6500b6d))
+
+
+### Bug Fixes
+
+* ask to return value ([#2659](https://github.com/windmill-labs/windmill/issues/2659)) ([de1e1f5](https://github.com/windmill-labs/windmill/commit/de1e1f545d4cd42f46d9af9a0349af86acf1901c))
+* fix embedded approval step timeouts ([51ce2f8](https://github.com/windmill-labs/windmill/commit/51ce2f8cb308da285dab0dc433bf596caa5eeed0))
+* fix error handling for list of errors ([c1bb97d](https://github.com/windmill-labs/windmill/commit/c1bb97d990810c9b3d909c9a045f0ce84be6c25e))
+* set session code completion to enabled by default ([#2664](https://github.com/windmill-labs/windmill/issues/2664)) ([ded0bb8](https://github.com/windmill-labs/windmill/commit/ded0bb890bb54ef80c857395474e692865ee4717))
+
+## [1.212.0](https://github.com/windmill-labs/windmill/compare/v1.211.0...v1.212.0) (2023-11-20)
+
+
+### Features
+
+* S3 file picker as a drawer ([#2640](https://github.com/windmill-labs/windmill/issues/2640)) ([624b4d0](https://github.com/windmill-labs/windmill/commit/624b4d0e9898dddcce3cb2ce989ce1f9e4736061))
+* test an iteration ([c2598b3](https://github.com/windmill-labs/windmill/commit/c2598b330450f9885f7d10e2b5baa54d6ef88cc5))
+* upgrade to gpt-4-turbo ([#2655](https://github.com/windmill-labs/windmill/issues/2655)) ([8ea98c2](https://github.com/windmill-labs/windmill/commit/8ea98c2c8d636209954a267116eb03ab13217ef8))
+
+## [1.211.0](https://github.com/windmill-labs/windmill/compare/v1.210.1...v1.211.0) (2023-11-20)
+
+
+### Features
+
+* agent mode v0 ([cd260e7](https://github.com/windmill-labs/windmill/commit/cd260e7062802ab39115025577a2456c66435899))
+* agent mode v0 ([47ad8d6](https://github.com/windmill-labs/windmill/commit/47ad8d6013e7144a5d018f43c827b171228b5f42))
+* ai regex ([19907e4](https://github.com/windmill-labs/windmill/commit/19907e4012e88a8fd28a5f0564a7ea48ec12020c))
+* custom error messages for forms ([2f15ebc](https://github.com/windmill-labs/windmill/commit/2f15ebc5f9628b0d26ec08b3527ddc96c6d42ba0))
+* **python:** Add functionality and resiliency to wmill python client ([#2650](https://github.com/windmill-labs/windmill/issues/2650)) ([c7a30f7](https://github.com/windmill-labs/windmill/commit/c7a30f7c9db26252f0ee69e1276ddccc0d52acb3))
+
+
+### Bug Fixes
+
+* add refresh button to item picker ([4882d94](https://github.com/windmill-labs/windmill/commit/4882d94dfe18c156662fe483e1dd4f5d3e3be3af))
+* fix base64 assignment (file input) ([70f1210](https://github.com/windmill-labs/windmill/commit/70f121035edd94b3940f530a5603b1ff4bf03839))
+* main broken tests ([#2652](https://github.com/windmill-labs/windmill/issues/2652)) ([c90f7f1](https://github.com/windmill-labs/windmill/commit/c90f7f167e0b9c20008a62eeeef7819e24fc3da9))
+* token expiry is equal to timeout ([b667317](https://github.com/windmill-labs/windmill/commit/b667317d44f37bb50f24e356a1a3d231ebe8b4b4))
+
 ## [1.210.1](https://github.com/windmill-labs/windmill/compare/v1.210.0...v1.210.1) (2023-11-18)
 
 

Dockerfile

@@ -1,4 +1,8 @@
-FROM debian:bookworm-slim as nsjail
+ARG DEBIAN_IMAGE=debian:bookworm-slim
+ARG RUST_IMAGE=rust:slim-bookworm
+ARG PYTHON_IMAGE=python:3.11.4-slim-bookworm
+
+FROM ${DEBIAN_IMAGE} as nsjail
 
 WORKDIR /nsjail
 
@@ -22,7 +26,7 @@ RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
 RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
-FROM rust:slim-bookworm AS rust_base
+FROM ${RUST_IMAGE} AS rust_base
 
 RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
@@ -87,7 +91,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:bookworm-slim as downloader
+FROM ${DEBIAN_IMAGE} as downloader
 
 ARG TARGETPLATFORM
 
@@ -101,29 +105,44 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol
 
 RUN unzip deno.zip && rm deno.zip
 
-FROM python:3.11.4-slim-bookworm
+FROM ${PYTHON_IMAGE}
 
 ARG TARGETPLATFORM
-
+ARG POWERSHELL_VERSION=7.3.5
+ARG POWERSHELL_DEB_VERSION=7.3.5-1
+ARG RCLONE_VERSION=1.60.1
+ARG KUBECTL_VERSION=1.27.2
+ARG HELM_VERSION=3.12.0
 ARG APP=/usr/src/app
+ARG WITH_POWERSHELL=true
+ARG WITH_RCLONE=true
+ARG WITH_KUBECTL=true
+ARG WITH_HELM=true
+
 
 RUN apt-get update \
     && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential unixodbc xmlsec1 \
     && rm -rf /var/lib/apt/lists/*
 
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then apt-get update -y && apt install libicu-dev -y && wget -O 'pwsh.deb' 'https://github.com/PowerShell/PowerShell/releases/download/v7.3.5/powershell_7.3.5-1.deb_amd64.deb' && \
-    dpkg --install 'pwsh.deb' && \
-    rm 'pwsh.deb'; else echo 'pwshell not on amd64'; fi
+RUN if [ "$WITH_POWERSHELL" = "true" ]; then \
+         if [ "$TARGETPLATFORM" = "linux/amd64" ]; then apt-get update -y && apt install libicu-dev -y && wget -O 'pwsh.deb' "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/powershell_${POWERSHELL_DEB_VERSION}.deb_amd64.deb" && \
+         dpkg --install 'pwsh.deb' && \
+         rm 'pwsh.deb'; else echo 'pwshell not on amd64'; fi;  \
+     else echo 'Building the image without powershell'; fi
 
-RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
-    wget https://get.helm.sh/helm-v3.12.0-linux-$arch.tar.gz && \
-    tar -zxvf helm-v3.12.0-linux-$arch.tar.gz  && \
-    mv linux-$arch/helm /usr/local/bin/helm &&\
-    chmod +x /usr/local/bin/helm
+RUN if [ "$WITH_HELM" = "true" ]; then \
+        arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+        wget "https://get.helm.sh/helm-v${HELM_VERSION}-linux-$arch.tar.gz" && \
+        tar -zxvf "helm-v${HELM_VERSION}-linux-$arch.tar.gz"  && \
+        mv linux-$arch/helm /usr/local/bin/helm &&\
+        chmod +x /usr/local/bin/helm; \
+    else echo 'Building the image without helm'; fi
 
-RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
-    curl -LO "https://dl.k8s.io/release/v1.27.2/bin/linux/$arch/kubectl" && \
-    install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+RUN if [ "$WITH_KUBECTL" = "true" ]; then \
+        arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+        curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/$arch/kubectl" && \
+        install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl; \
+    else echo 'Building the image without kubectl'; fi
 
 RUN set -eux; \
     arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
@@ -140,22 +159,25 @@ RUN set -eux; \
     unzip awscliv2.zip && \
     ./aws/install && rm awscliv2.zip
 
-RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
-    curl -o rclone.zip "https://downloads.rclone.org/v1.60.1/rclone-v1.60.1-linux-$arch.zip"; \
-    unzip -p rclone.zip rclone-v1.60.1-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
-    chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone
+RUN if [ "$WITH_RCLONE" = "true" ]; then \
+        arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
+        curl -o rclone.zip "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-$arch.zip"; \
+        unzip -p rclone.zip rclone-v${RCLONE_VERSION}-linux-$arch/rclone > /usr/bin/rclone; rm rclone.zip; \
+        chown root:root /usr/bin/rclone; chmod 755 /usr/bin/rclone; \
+    else echo 'Building the image without rclone'; fi
+
 
 RUN set -eux; \
     arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
     case "$arch" in \
     'amd64') \
-    targz='go1.21.0.linux-amd64.tar.gz'; \
+    targz='go1.21.5.linux-amd64.tar.gz'; \
     ;; \
     'arm64') \
-    targz='go1.21.0.linux-arm64.tar.gz'; \
+    targz='go1.21.5.linux-arm64.tar.gz'; \
     ;; \
     'armhf') \
-    targz='go1.21.0.linux-armv6l.tar.gz'; \
+    targz='go1.21.5.linux-armv6l.tar.gz'; \
     ;; \
     *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
     esac; \

README.md

@@ -1,14 +1,11 @@
 <p align="center">
   <a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
 </p>
-<p align="center">
-    <em>.</em>
-</p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
+Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
 
 <p align=center>
-Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, TypeScript, Go, Bash, SQL. Scripts can be generated by an AI assistant powered by OpenAI.
+Scripts are turned into UIs and no-code modules, no-code modules can be composed into rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL.
 </p>
 
 <p align="center">
@@ -36,7 +33,7 @@ Scripts are turned into UIs and no-code modules, no-code modules can be composed
   <a href="https://app.windmill.dev">Try it</a> - <a href="https://www.windmill.dev/docs/intro/">Docs</a> - <a href="https://discord.gg/V7PM2YHsPB">Discord</a> - <a href="https://hub.windmill.dev">Hub</a> - <a href="https://www.windmill.dev/docs/misc/contributing">Contributor's guide</a>
 </p>
 
-# Windmill - Turn scripts into workflows and UIs that you can share and run at scale
+# Windmill - Developer platform for APIs, background jobs, workflows and UIs
 
 Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
 dedicated instance and commercial support and licenses.
@@ -45,12 +42,13 @@ dedicated instance and commercial support and licenses.
 
 https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252
 
-- [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
+- [Windmill - Developer platform for APIs, background jobs, workflows and UIs](#windmill---developer-platform-for-apis-background-jobs-workflows-and-uis)
   - [Main Concepts](#main-concepts)
   - [Show me some actual script code](#show-me-some-actual-script-code)
   - [CLI](#cli)
     - [Running scripts locally](#running-scripts-locally)
   - [Stack](#stack)
+  - [Fastest Self-Hostable Workflow Engine](#fastest-self-hostable-workflow-engine)
   - [Security](#security)
     - [Sandboxing](#sandboxing)
     - [Secrets, credentials and sensitive values](#secrets-credentials-and-sensitive-values)
@@ -59,12 +57,10 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
   - [How to self-host](#how-to-self-host)
     - [Docker compose](#docker-compose)
     - [Kubernetes (k8s) and Helm charts](#kubernetes-k8s-and-helm-charts)
-    - [Postgres without superuser](#postgres-without-superuser)
+    - [Run from binaries](#run-from-binaries)
+    - [OAuth, SSO \& SMTP](#oauth-sso--smtp)
     - [Commercial license](#commercial-license)
-    - [OAuth for self-hosting](#oauth-for-self-hosting)
-    - [smtp for self-hosting](#smtp-for-self-hosting)
-    - [Resource types](#resource-types)
-    - [Manually fetch latest Windmill binary](#manually-fetch-latest-windmill-binary)
+    - [Integrations](#integrations)
   - [Environment Variables](#environment-variables)
   - [Run a local dev setup](#run-a-local-dev-setup)
     - [only Frontend](#only-frontend)
@@ -75,22 +71,28 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
 ## Main Concepts
 
 1. Define a minimal and generic script in Python, TypeScript, Go or Bash that
-   solves a specific task. Here sending a POST request. The code can be defined
-   in the provided Web IDE or synchronized with your own github repo:
+   solves a specific task. The code can be defined
+   in the [provided Web IDE](https://www.windmill.dev/docs/code_editor) or [synchronized with your own GitHub repo](https://www.windmill.dev/docs/advanced/cli/sync) (e.g. through [VS Code](https://www.windmill.dev/docs/cli_local_dev/vscode-extension) extension):
+
    ![Step 1](./imgs/windmill-editor.png)
 
-2. Your scripts parameters are automatically parsed and generate a frontend.
-   ![Step 2](./imgs/windmill-run.png) ![Step 3](./imgs/windmill-result.png)
+2. Your scripts parameters are automatically parsed and [generate a frontend](https://www.windmill.dev/docs/core_concepts/auto_generated_uis).
 
-3. Make it flow! You can chain your scripts or scripts made by the community
+  ![Step 2](./imgs/windmill-run.png)
+  
+  ![Step 3](./imgs/windmill-result.png)
+
+3. Make it [flow](https://www.windmill.dev/docs/flows/flow_editor)! You can chain your scripts or scripts made by the community
    shared on [WindmillHub](https://hub.windmill.dev).
+
    ![Step 3](./imgs/windmill-flow.png)
 
-4. Build complex UIs on top of your scripts and flows.
+4. Build [complex UIs](https://www.windmill.dev/docs/apps/app_editor) on top of your scripts and flows.
+
    ![Step 4](./imgs/windmill-builder.png)
 
-Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
-through webhooks.
+Scripts and flows can also be triggered by a [cron schedule](https://www.windmill.dev/docs/core_concepts/scheduling) (e.g. '_/5 _ \* \* \*') or
+through [webhooks](https://www.windmill.dev/docs/core_concepts/webhooks).
 
 You can build your entire infra on top of Windmill!
 
@@ -146,7 +148,9 @@ instance from local commands. See
 You can run your script locally easily, you simply need to pass the right
 environment variables for the `wmill` client library to fetch resources and
 variables from your instance if necessary. See more:
-<https://www.windmill.dev/docs/advanced/local_development/>.
+<https://www.windmill.dev/docs/advanced/local_development>.
+
+To develop & test locally scripts & flows, we recommend using the Windmill VS Code extension: <https://www.windmill.dev/docs/cli_local_dev/vscode-extension>.
 
 ## Stack
 
@@ -167,6 +171,14 @@ variables from your instance if necessary. See more:
 - python runtime is python3
 - golang runtime is 1.19.1
 
+## Fastest Self-Hostable Workflow Engine
+
+We have compared Windmill to other self-hostable workflow engines (Airflow, Prefect & Temporal) and Windmill is the most performant solution for both benchmarks: one flow composed of 40 lightweight tasks & one flow composed of 10 long-running tasks.
+
+All methodology & results on our [Benchmarks](https://www.windmill.dev/docs/misc/benchmarks/competitors#airflow-setup) page.
+
+![Fastest workflow engine](./imgs/fastest.png)
+
 ## Security
 
 ### Sandboxing
@@ -204,10 +216,14 @@ back to the database is ~50ms. A typical lightweight deno job will take around
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://www.windmill.dev/docs/advanced/self_host).
+[Self-Host documentation](https://www.windmill.dev/docs/advanced/self_host).
 
 ### Docker compose
 
+Windmill can be deployed using 3 files: ([docker-compose.yml](./docker-compose.yml), [Caddyfile](./Caddyfile) and a [.env](./.env)) in a single command.
+
+Make sure Docker is started, and run:
+
 ```
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
@@ -222,6 +238,8 @@ The default super-admin user is: admin@windmill.dev / changeme.
 
 From there, you can follow the setup app and create other users.
 
+More details in [Self-Host Documention](https://www.windmill.dev/docs/advanced/self_host#docker).
+
 ### Kubernetes (k8s) and Helm charts
 
 We publish helm charts at:
@@ -247,14 +265,16 @@ Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin
 settings. Do note that there is a limit of 50 SSO users on the community
 edition.
 
+[See documentation](https://www.windmill.dev/docs/misc/setup_oauth).
+
 ### Commercial license
 
-To self-host Windmill, you must respect the terms of the AGPLv3 license which
+To self-host Windmill, you must respect the terms of the [AGPLv3 license](https://www.gnu.org/licenses/agpl-3.0.en.html) which
 you do not need to worry about for personal uses. For business uses, you should
-be fine if you do not re-expose it in any way Windmill to your users and are
+be fine if you do not re-expose Windmill in any way to your users and are
 comfortable with AGPLv3.
 
-To re-expose any Windmill parts to your users as a feature of your product, or
+To [re-expose any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling) as a feature of your product, or
 to build a feature on top of Windmill, to comply with AGPLv3 your product must
 be AGPLv3 or you must get a commercial license. Contact us at
 <ruben@windmill.dev> if you have any doubts.
@@ -264,9 +284,11 @@ your current infrastructure to Windmill, support with tight SLA, and our global
 cache sync for high-performance/no dependency cache miss of cluster from 10+
 nodes to 200+ nodes.
 
-### Resource types
+### Integrations
 
-You will also want to import all the approved resource types from
+In Windmill, integrations are referred to as [resources and resource types](https://www.windmill.dev/docs/core_concepts/resources_and_types). Each Resource has a Resource Type that defines the schema that the resource needs to implement.
+
+On self-hosted instances, you might want to import all the approved resource types from
 [WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
 it being synced automatically everyday.
 
@@ -341,11 +363,11 @@ it being synced automatically everyday.
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
 | CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                       | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
-| GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
+| GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script or flow to run when a root job fails. The path needs to be prefixed with either `script/` or `flow/` to indicate the kind of error handler being used (assuming `script/` by default). The error handler will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |
 | SECRET_SALT                                   | None                                       | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server    | Server + Worker       |
-| OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI API base path (no trailing slash)                                                                                                                                                     | Server                |
+| OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI path to be used instead of the OpenAI path. All Windmill AI features will run on the specified deployed model. Format: `https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}`     | Server                |
 | DISABLE_EMBEDDING                             | false                                      | Disable local embedding search of hub scripts                                                                                                                                                      | Server                |
 | DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |

backend/.sqlx/query-0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where hash = $1 AND workspace_id = $2",
+  "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -37,7 +37,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",
@@ -59,6 +58,11 @@
         "ordinal": 6,
         "name": "priority",
         "type_info": "Int2"
+      },
+      {
+        "ordinal": 7,
+        "name": "delete_after_use",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -74,8 +78,9 @@
       true,
       false,
       true,
+      true,
       true
     ]
   },
-  "hash": "d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071"
+  "hash": "0234079d04372b82ec8f2990916433bd8771c21d22a1db02eb1e388a8bfb28ff"
 }

backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n            WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90"
+}

backend/.sqlx/query-0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                    SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n                    WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0c0b5d5d1e6ab2fed7532f94b50be3210e3845b61551691bbef81c2b6fb01121"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -87,6 +87,11 @@
         "ordinal": 16,
         "name": "large_file_storage",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 17,
+        "name": "git_sync",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -111,6 +116,7 @@
       false,
       true,
       false,
+      true,
       true
     ]
   },

backend/.sqlx/query-1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "restart_unless_cancelled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838"
+}

backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json

@@ -45,7 +45,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }
@@ -67,7 +68,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n                WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "26106be4d94c159cc8d9eb37a3b94927c5d3d43c18cdb5e98e979e761ed6ed0e"
+}

backend/.sqlx/query-2a3ebe1b0eae5b2164894321e138cc4dc0293788aeb98d05d95d18dfc708d6a6.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n             WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "2a3ebe1b0eae5b2164894321e138cc4dc0293788aeb98d05d95d18dfc708d6a6"
-}

backend/.sqlx/query-322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow_status->'cleanup_module' as cleanup_module\n        FROM queue\n        WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "cleanup_module",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "322038aae939d738f47b5bddaec2a6342e25e815dcd8b9fd620ddb81063c5d76"
+}

backend/.sqlx/query-362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by created_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "deployment_msg",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675"
+}

backend/.sqlx/query-3c83e6adf21b7a8572529f9175cf6d2d30ea8c921ae2bf751b6825a3e453a012.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                UPDATE queue\n                   SET suspend = $1, suspend_until = now() + interval '14 day', running = true\n                 WHERE id = $2\n                ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "3c83e6adf21b7a8572529f9175cf6d2d30ea8c921ae2bf751b6825a3e453a012"
-}

backend/.sqlx/query-42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE completed_job\n                    SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb\n                    WHERE id = ANY($1)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "UuidArray"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "42543450a5c1b988b258fb1e0c00cd0a53a0395bc60e35817ff149c4f27a19fb"
+}

backend/.sqlx/query-47a74719faba8a6377ba157d25929a6f1e74afdffd642d5a1c7db57e745e0c01.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n             WHERE id = $3\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "47a74719faba8a6377ba157d25929a6f1e74afdffd642d5a1c7db57e745e0c01"
-}

backend/.sqlx/query-4de3ba6930b91c639f9f3939dab7b3ab35aa792484e4ccb8084c336a2ec60f45.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET flow_status = JSONB_SET(flow_status, ARRAY['step'], $1)\n             WHERE id = $2\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4de3ba6930b91c639f9f3939dab7b3ab35aa792484e4ccb8084c336a2ec60f45"
-}

backend/.sqlx/query-51d253d3cba9e2a36c291f6f9cf7023834484e89d91f0267d9dcaaafe406c64c.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "51d253d3cba9e2a36c291f6f9cf7023834484e89d91f0267d9dcaaafe406c64c"
+}

backend/.sqlx/query-53ec897ac019abf3fc4499f593adadf85af465d66c1a23b0549d176edbabad79.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                UPDATE queue\n                   SET flow_status = JSONB_SET(\n                                     JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2),\n                                                            ARRAY['step'], $3)\n                 WHERE id = $4\n                  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "53ec897ac019abf3fc4499f593adadf85af465d66c1a23b0549d176edbabad79"
-}

backend/.sqlx/query-53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                    SET suspend = $1, suspend_until = now() + interval '14 day', running = true\n                    WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "53ff0e14c35a3e84585a699e55093546db5c5f0ad0c5f92f34aaf2cdd125d130"
+}

backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json

@@ -87,6 +87,11 @@
         "ordinal": 16,
         "name": "large_file_storage",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 17,
+        "name": "git_sync",
+        "type_info": "Jsonb"
       }
     ],
     "parameters": {
@@ -111,6 +116,7 @@
       false,
       true,
       false,
+      true,
       true
     ]
   },

backend/.sqlx/query-5d1ec728380ea8baf64df54743e73008ee9a58e6f39a5bf31a2ed099727f5c04.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM worker_ping WHERE custom_tags @> $1 AND ping_at > now() - interval '1 minute')",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "5d1ec728380ea8baf64df54743e73008ee9a58e6f39a5bf31a2ed099727f5c04"
+}

backend/.sqlx/query-61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                    SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2)\n                    WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "61656548991bf6d44c839373cb3e29d3ca1170a2ac0d3dce0b5df0e8677a4874"
+}

backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json

@@ -40,7 +40,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }
@@ -60,7 +61,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "652835b2b7f801532a591988ac76d385188991c6654d529f6d65f6f03794844a"
+}

backend/.sqlx/query-6bb4fab9e74c421fb30cfd856a1a9b44c76f1cf1485ceba50aeede9daec94c01.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6bb4fab9e74c421fb30cfd856a1a9b44c76f1cf1485ceba50aeede9daec94c01"
+}

backend/.sqlx/query-7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -28,7 +28,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",
@@ -62,10 +61,12 @@
         "Int4",
         "Bool",
         "Bool",
-        "Int2"
+        "Int2",
+        "Bool",
+        "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db"
+  "hash": "7db59e88a10150ab6a3e81f164343e6bb2be6149fb690e63b8a0c376f39e9924"
 }

backend/.sqlx/query-7f9f1ce221835fc3ff6c864c362519fde3d08e496119e51d8ed4fe65792a85c1.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7f9f1ce221835fc3ff6c864c362519fde3d08e496119e51d8ed4fe65792a85c1"
+}

backend/.sqlx/query-8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                        SET leaf_jobs = JSONB_SET(coalesce(leaf_jobs, '{}'::jsonb), ARRAY[$1::TEXT], $2)\n                        WHERE COALESCE((SELECT root_job FROM queue WHERE id = $3), $3) = id",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8929150cb9262623eb12c908cb96c4d7e8cac594900ab0956c098e1cfa75e2f0"
+}

backend/.sqlx/query-8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE completed_job\n        SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb\n        WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8a44ca0cfe1e154138cbbf5ebddebbba2e1149d2e329d1d89eecc1c008c93a31"
+}

backend/.sqlx/query-8e0679c2b1bd451691fe5c69a2841ddc9f211311316ec6b9d4699b2c70997a19.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, dedicated_worker, value->>'early_return' as early_return from flow WHERE path = $1 and workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "early_return",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      null
+    ]
+  },
+  "hash": "8e0679c2b1bd451691fe5c69a2841ddc9f211311316ec6b9d4699b2c70997a19"
+}

backend/.sqlx/query-9681c348429f5e647ab55a4a54933faa64018f8539a143641f10d14ddd3a09c4.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n             WHERE id = $2\n             RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "9681c348429f5e647ab55a4a54933faa64018f8539a143641f10d14ddd3a09c4"
-}

backend/.sqlx/query-9ce6eecfa10c2f71cc536957ead498ad829ea8023ce449eab27225ec66738525.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1\n         ORDER BY created_at DESC",
+  "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1\n            ORDER BY created_at DESC",
   "describe": {
     "columns": [
       {
@@ -48,5 +48,5 @@
       true
     ]
   },
-  "hash": "773c145013623e4eb29a8df70e46d805dc5b15942a36a0d988521abb0cb34e41"
+  "hash": "9ce6eecfa10c2f71cc536957ead498ad829ea8023ce449eab27225ec66738525"
 }

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -24,7 +24,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }
@@ -40,7 +41,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-a6145b0482c9e5da245059a80b1563cad20318fd2dd8aef33f9ca97de1826b8b.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET flow_status = JSONB_SET(flow_status, ARRAY['failure_module'], $1)\n             WHERE id = $2\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "a6145b0482c9e5da245059a80b1563cad20318fd2dd8aef33f9ca97de1826b8b"
-}

backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c"
+}

backend/.sqlx/query-a96dd57b127a1adbdca13867a76d7fd9f9a9c76b02bcebd2830d928821f3abc0.json

@@ -1,16 +1,15 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET canceled = true, canceled_reason = 'http connection broke', canceled_by = $1 WHERE id = $2 AND workspace_id = $3",
+  "query": "UPDATE queue SET canceled = true, canceled_reason = 'http connection broke', canceled_by = queue.created_by WHERE id = $1 AND workspace_id = $2",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
-        "Varchar",
         "Uuid",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "460a3c3161899e172d2d20004e12ad3f331c73f260d099749a9e8fb85ae5614e"
+  "hash": "a96dd57b127a1adbdca13867a76d7fd9f9a9c76b02bcebd2830d928821f3abc0"
 }

backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json

@@ -30,7 +30,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }
@@ -46,7 +47,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-abb56dec024600d0d7a63212f63c3296196898003707f2cb3a890690173a51a5.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path FROM app WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "abb56dec024600d0d7a63212f63c3296196898003707f2cb3a890690173a51a5"
+}

backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                    SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \n                    workspace_id = $2)\n                    ",
+  "query": "\n                    SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND\n                    workspace_id = $2)\n                    ",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       false
     ]
   },
-  "hash": "d581bfb507d3a875f07677187717a469ec6ab1db1585835c961409612d81ec7d"
+  "hash": "ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145"
 }

backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n            WHERE id = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9"
+}

backend/.sqlx/query-b83e58972d8f8285eb604de94c3d363102f5f6705d6debb442a85126ed59d863.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b83e58972d8f8285eb604de94c3d363102f5f6705d6debb442a85126ed59d863"
+}

backend/.sqlx/query-bc1f2f169b4960dae02f62e37bb4ae081146e598109860ee1f42cd3778c5ebaf.json

@@ -0,0 +1,52 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script'\n             ORDER BY created_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "label",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "token_prefix",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "expiration",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "last_used_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 5,
+        "name": "scopes",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      null,
+      true,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "bc1f2f169b4960dae02f62e37bb4ae081146e598109860ee1f42cd3778c5ebaf"
+}

backend/.sqlx/query-c461bdf083d7aaacafb6c981556b218342626524ee0c92f083dbff2930b7de9e.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                UPDATE queue\n                   SET flow_status = JSONB_SET(\n                                     JSONB_SET(flow_status, ARRAY['failure_module'], $1),\n                                                            ARRAY['step'], $2)\n                 WHERE id = $3\n                  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "c461bdf083d7aaacafb6c981556b218342626524ee0c92f083dbff2930b7de9e"
-}

backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg\n        FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n        WHERE a.workspace_id = $1 AND a.path = $2\n        ORDER BY created_at DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "app_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "version_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "deployment_msg",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f"
+}

backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43"
+}

backend/.sqlx/query-db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "db558b5ecdc4c3b1af0def511f1bcd91a548f00376f644c8ba38f73812b462d0"
+}

backend/.sqlx/query-e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where hash = $1 AND workspace_id = $2",
+  "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",
@@ -64,6 +63,11 @@
         "ordinal": 7,
         "name": "priority",
         "type_info": "Int2"
+      },
+      {
+        "ordinal": 8,
+        "name": "delete_after_use",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -80,8 +84,9 @@
       true,
       false,
       true,
+      true,
       true
     ]
   },
-  "hash": "11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c"
+  "hash": "e541faa52c530c787d45c5ee70930687587c139606056bdc06cd860abeb0805b"
 }

backend/.sqlx/query-e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                      SELECT success\n                        FROM completed_job\n                       WHERE id = ANY($1)\n                        ",
+  "query": "SELECT success FROM completed_job WHERE id = ANY($1)",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       false
     ]
   },
-  "hash": "ed400dbf4a1fa597bc984240cfe9069369a8a265b127116187d78c0ba93ae076"
+  "hash": "e58cf2e3deb9aa1e9f37313a33e5d44fcfdb40a4764e1dc34896bd85b3d007f9"
 }

backend/.sqlx/query-e5a130478c8a581412f786b430c2c166c489faccbd0d9b448f8e7ad57559e605.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            UPDATE queue\n               SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n             WHERE id = $2\n             RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "e5a130478c8a581412f786b430c2c166c489faccbd0d9b448f8e7ad57559e605"
-}

backend/.sqlx/query-ec4f70d9e6392ade016245de73e7a2831d430838307b086c96511f91e76a214a.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ec4f70d9e6392ade016245de73e7a2831d430838307b086c96511f91e76a214a"
+}

backend/.sqlx/query-f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
+  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
   "describe": {
     "columns": [
       {
@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",
@@ -64,6 +63,11 @@
         "ordinal": 7,
         "name": "priority",
         "type_info": "Int2"
+      },
+      {
+        "ordinal": 8,
+        "name": "delete_after_use",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -80,8 +84,9 @@
       true,
       false,
       true,
+      true,
       true
     ]
   },
-  "hash": "b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d"
+  "hash": "f760667ffd6727443f023480627f62728bf9b6284d92e5fa9b97dc4e0b1c3a93"
 }

backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json

@@ -39,7 +39,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }
@@ -61,7 +62,8 @@
                 "graphql",
                 "postgresql",
                 "noop",
-                "appdependencies"
+                "appdependencies",
+                "deploymentcallback"
               ]
             }
           }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.210.1"
+version = "1.224.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -21,7 +21,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.210.1"
+version = "1.224.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -35,6 +35,7 @@ incremental = true
 
 [features]
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+saml = ["windmill-api/saml"]
 benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
@@ -97,8 +98,8 @@ headers = "^0"
 hyper = { version = "^0", features = ["full"] }
 tokio = { version = "^1", features = ["full", "tracing"] }
 tower = "^0"
-tower-http = { version = "^0", features = ["trace", "cors"] }
-tower-cookies = "^0"
+tower-http = { version = "^0.4", features = ["trace", "cors"] }
+tower-cookies = "0.9.0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
@@ -137,6 +138,7 @@ json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
 deno_fetch = "0.139.0"
+deno_tls = "0.102.0"
 deno_console = "0.115.0"
 deno_url = "0.115.0"
 deno_webidl = "0.115.0"
@@ -180,7 +182,7 @@ async-stripe = { version = "0.25.2", features = [
 async_zip = { version = "0.0.11", features = ["full"] }
 once_cell = "1.17.1"
 rsmq_async = { version = "5.1.5" }
-gosyn = "0.2.2"
+gosyn = "0.2.6"
 bytes = "1.4.0"
 gethostname = "0.4.3"
 wasm-bindgen = "0.2"
@@ -192,7 +194,7 @@ tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-
 mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls"]}
 postgres-native-tls = "^0"
 native-tls = "^0"
-samael = { version = "0.0.12", features = ["xmlsec"] }
+samael = { version = "0.0.14", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
 rust_decimal = { version = "^1", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"

backend/migrations/20231122100516_relax_tag_varchars.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231122100516_relax_tag_varchars.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE queue ALTER COLUMN tag TYPE varchar(255);
+ALTER TABLE completed_job ALTER COLUMN tag TYPE varchar(255);

backend/migrations/20231128105015_add_reports_worker_group.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231128105015_add_reports_worker_group.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+INSERT INTO config (name, config) VALUES  
+    ('worker__reports', '{"init_bash": "apt-get update\napt-get install -y chromium", "worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "chromium"]}') ON CONFLICT DO NOTHING;

backend/migrations/20231129164511_git_repo_workspace_setting.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN git_sync;
+
+DROP TABLE IF EXISTS deployment_metadata;

backend/migrations/20231129164511_git_repo_workspace_setting.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'deploymentcallback';
+ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS git_sync JSONB;
+
+CREATE TABLE deployment_metadata(
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    path VARCHAR(255) NOT NULL,
+    script_hash BIGINT,
+    app_version BIGSERIAL,
+    callback_job_ids UUID[],
+    deployment_msg TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_script ON deployment_metadata (workspace_id, script_hash) WHERE script_hash IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_flow ON deployment_metadata (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_app ON deployment_metadata (workspace_id, path, app_version) WHERE app_version IS NOT NULL;

backend/migrations/20231203110120_add_more_script_config.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE script DROP COLUMN timeout;
+ALTER TABLE script DROP COLUMN delete_after_use;
+ALTER TABLE script DROP COLUMN restart_unless_cancelled;
+ALTER TABLE flow DROP COLUMN timeout;

backend/migrations/20231203110120_add_more_script_config.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN timeout INTEGER;
+ALTER TABLE flow ADD COLUMN timeout INTEGER;
+ALTER TABLE script ADD COLUMN delete_after_use BOOLEAN;
+ALTER TABLE script ADD COLUMN restart_unless_cancelled BOOLEAN;

backend/migrations/20231204144423_refactor_deployment_metadata_table.down.sql

@@ -0,0 +1,15 @@
+-- Add down migration script here
+DROP TABLE deployment_metadata;
+
+CREATE TABLE deployment_metadata(
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    path VARCHAR(255) NOT NULL,
+    script_hash BIGINT,
+    app_version BIGINT,
+    callback_job_ids UUID[],
+    deployment_msg TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_script ON deployment_metadata (workspace_id, script_hash) WHERE script_hash IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_flow ON deployment_metadata (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_app ON deployment_metadata (workspace_id, path, app_version) WHERE app_version IS NOT NULL;

backend/migrations/20231204144423_refactor_deployment_metadata_table.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+DROP TABLE deployment_metadata;
+
+CREATE TABLE deployment_metadata(
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    path VARCHAR(255) NOT NULL,
+    script_hash BIGINT,
+    app_version BIGINT,
+    callback_job_ids UUID[],
+    deployment_msg TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_script ON deployment_metadata (workspace_id, script_hash) WHERE script_hash IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_flow ON deployment_metadata (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_app ON deployment_metadata (workspace_id, path, app_version) WHERE app_version IS NOT NULL;

backend/migrations/20231208141824_add_chromium_tag.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231208141824_add_chromium_tag.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+INSERT INTO global_settings VALUES ('custom_tags', '["chromium"]')
+ON CONFLICT (name) DO UPDATE SET value = jsonb_insert(global_settings.value, '{0}', '"chromium"', true)
+WHERE NOT global_settings.value @> '["chromium"]';
+
+

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -45,7 +45,8 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "shapefile" => "pyshp",
     "sklearn" => "scikit-learn",
     "umap" => "umap-learn",
-    "cv2" => "opencv-python"
+    "cv2" => "opencv-python",
+    "atlassian" => "atlassian-python-api"
 };
 
 fn replace_import(x: String) -> String {
@@ -167,7 +168,7 @@ pub async fn parse_python_imports(
                 let code = sqlx::query_scalar!(
                     r#"
                     SELECT content FROM script WHERE path = $1 AND workspace_id = $2
-                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND 
+                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND
                     workspace_id = $2)
                     "#,
                     &rpath,

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -61,24 +61,39 @@ lazy_static::lazy_static! {
     static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+(?:\[\])?))?"#).unwrap();
 
     // -- $1 name (type) = default
-    static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
+    static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
+    pub static ref RE_ARG_MYSQL_NAMED: Regex = Regex::new(r#"(?m)^-- :([a-z_][a-z0-9_]*) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
-    static ref RE_ARG_PGSQL: Regex = Regex::new(r#"(?m)^-- \$(\d+) (\w+)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
+    static ref RE_ARG_PGSQL: Regex = Regex::new(r#"(?m)^-- \$(\d+) (\w+)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
     // -- @name (type) = default
-    static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
+    static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
-    static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
+    static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
 
-    static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
+    static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
 
 }
 
+fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value> {
+    match parsed_typ {
+        _ if default.to_lowercase() == "null" => None,
+        Typ::Int => default.parse::<i64>().ok().map(|x| json!(x)),
+        Typ::Float => default.parse::<f64>().ok().map(|x| json!(x)),
+        Typ::Bool => default.parse::<bool>().ok().map(|x| json!(x)),
+        Typ::Str(_) if default.len() >= 2 && default.starts_with("'") && default.ends_with("'") => {
+            Some(json!(&default[1..default.len() - 1]))
+        }
+        _ => Some(json!(default)),
+    }
+}
 fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     let mut args: Vec<Arg> = vec![];
 
-    for cap in RE_ARG_MYSQL.captures_iter(code) {
+    let mut using_named_args = false;
+    for cap in RE_ARG_MYSQL_NAMED.captures_iter(code) {
+        using_named_args = true;
         let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
         let typ = cap
             .get(2)
@@ -88,11 +103,7 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         let has_default = default.is_some();
         let parsed_typ = parse_mysql_typ(typ.as_str());
 
-        let parsed_default = default.and_then(|x| match parsed_typ {
-            Typ::Int => x.parse::<i64>().ok().map(|x| json!(x)),
-            Typ::Float => x.parse::<f64>().ok().map(|x| json!(x)),
-            _ => Some(json!(x)),
-        });
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
         args.push(Arg {
             name,
             typ: parsed_typ,
@@ -102,6 +113,30 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         });
     }
 
+    if !using_named_args {
+        // backwards compatibility
+        for cap in RE_ARG_MYSQL.captures_iter(code) {
+            let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
+            let typ = cap
+                .get(2)
+                .map(|x| x.as_str().to_string().to_lowercase())
+                .unwrap();
+            let default = cap.get(3).map(|x| x.as_str().to_string());
+            let has_default = default.is_some();
+            let parsed_typ = parse_mysql_typ(typ.as_str());
+
+            let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
+            args.push(Arg {
+                name,
+                typ: parsed_typ,
+                default: parsed_default,
+                otyp: Some(typ),
+                has_default,
+            });
+        }
+    }
+
     Ok(Some(args))
 }
 
@@ -141,11 +176,8 @@ fn parse_pg_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         let default = cap.get(3).map(|x| x.as_str().to_string());
         let has_default = default.is_some();
         let oarg = args[(i.unwrap() - 1) as usize].clone();
-        let parsed_default = default.and_then(|x| match oarg.typ {
-            Typ::Int => x.parse::<i64>().ok().map(|x| json!(x)),
-            Typ::Float => x.parse::<f64>().ok().map(|x| json!(x)),
-            _ => Some(json!(x)),
-        });
+        let parsed_default = default.and_then(|x| parsed_default(&oarg.typ, x));
+
         args[(i.unwrap() - 1) as usize] =
             Arg { name, typ: oarg.typ, default: parsed_default, otyp: oarg.otyp, has_default };
     }
@@ -166,11 +198,8 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         let has_default = default.is_some();
         let parsed_typ = parse_bigquery_typ(typ.as_str());
 
-        let parsed_default = default.and_then(|x| match parsed_typ {
-            Typ::Int => x.parse::<i64>().ok().map(|x| json!(x)),
-            Typ::Float => x.parse::<f64>().ok().map(|x| json!(x)),
-            _ => Some(json!(x)),
-        });
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
         args.push(Arg {
             name,
             typ: parsed_typ,
@@ -196,11 +225,8 @@ fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         let has_default = default.is_some();
         let parsed_typ = parse_snowflake_typ(typ.as_str());
 
-        let parsed_default = default.and_then(|x| match parsed_typ {
-            Typ::Int => x.parse::<i64>().ok().map(|x| json!(x)),
-            Typ::Float => x.parse::<f64>().ok().map(|x| json!(x)),
-            _ => Some(json!(x)),
-        });
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
         args.push(Arg {
             name,
             typ: parsed_typ,
@@ -226,11 +252,8 @@ fn parse_mssql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
         let has_default = default.is_some();
         let parsed_typ = parse_mssql_typ(typ.as_str());
 
-        let parsed_default = default.and_then(|x| match parsed_typ {
-            Typ::Int => x.parse::<i64>().ok().map(|x| json!(x)),
-            Typ::Float => x.parse::<f64>().ok().map(|x| json!(x)),
-            _ => Some(json!(x)),
-        });
+        let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
+
         args.push(Arg {
             name,
             typ: parsed_typ,
@@ -251,6 +274,7 @@ pub fn parse_mysql_typ(typ: &str) -> Typ {
         "int" | "uint" | "integer" => Typ::Int,
         "bool" | "bit" => Typ::Bool,
         "double precision" | "float" | "real" | "dec" | "fixed" => Typ::Float,
+        "date" | "datetime" | "timestamp" | "time" => Typ::Datetime,
         _ => Typ::Str(None),
     }
 }

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -311,6 +311,12 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
             } else {
                 let literals = types
                     .into_iter()
+                    .filter(|x| match ***x {
+                        TsType::TsKeywordType(TsKeywordType { kind, .. }) => {
+                            kind != TsKeywordTypeKind::TsStringKeyword
+                        }
+                        _ => true,
+                    })
                     .map(|x| match &**x {
                         TsType::TsLitType(TsLitType {
                             lit: TsLit::Str(Str { value, .. }), ..

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.202.1",
+  "version": "1.222.0",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -97,15 +97,6 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
-}
-
 function addHeapObject(obj) {
     if (heap_next === heap.length) heap.push(heap.length + 1);
     const idx = heap_next;
@@ -124,6 +115,15 @@ function getFloat64Memory0() {
     return cachedFloat64Memory0;
 }
 
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -518,6 +518,10 @@ function __wbg_get_imports() {
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
+    imports.wbg.__wbg_eval_1081105c41705556 = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -526,14 +530,6 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
-    imports.wbg.__wbg_eval_f742bccbdcf34b02 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -551,6 +547,10 @@ function __wbg_get_imports() {
         const ret = getObject(arg0) === getObject(arg1);
         return ret;
     };
+    imports.wbg.__wbindgen_bigint_from_u64 = function(arg0) {
+        const ret = BigInt.asUintN(64, arg0);
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_number_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'number' ? obj : undefined;
@@ -566,19 +566,19 @@ function __wbg_get_imports() {
         const ret = getObject(arg0) in getObject(arg1);
         return ret;
     };
-    imports.wbg.__wbindgen_bigint_from_u64 = function(arg0) {
-        const ret = BigInt.asUintN(64, arg0);
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
         return addHeapObject(ret);
     };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;
     };
-    imports.wbg.__wbg_get_4a9aa5157afeb382 = function(arg0, arg1) {
+    imports.wbg.__wbg_get_f01601b5a68d10e3 = function(arg0, arg1) {
         const ret = getObject(arg0)[arg1 >>> 0];
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_length_cace2e0b3ddc0502 = function(arg0) {
+    imports.wbg.__wbg_length_1009b1af0c481d7b = function(arg0) {
         const ret = getObject(arg0).length;
         return ret;
     };
@@ -586,39 +586,39 @@ function __wbg_get_imports() {
         const ret = typeof(getObject(arg0)) === 'function';
         return ret;
     };
-    imports.wbg.__wbg_next_15da6a3df9290720 = function(arg0) {
+    imports.wbg.__wbg_next_9b877f231f476d01 = function(arg0) {
         const ret = getObject(arg0).next;
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_next_1989a20442400aaa = function() { return handleError(function (arg0) {
+    imports.wbg.__wbg_next_6529ee0cca8d57ed = function() { return handleError(function (arg0) {
         const ret = getObject(arg0).next();
         return addHeapObject(ret);
     }, arguments) };
-    imports.wbg.__wbg_done_bc26bf4ada718266 = function(arg0) {
+    imports.wbg.__wbg_done_5fe336b092d60cf2 = function(arg0) {
         const ret = getObject(arg0).done;
         return ret;
     };
-    imports.wbg.__wbg_value_0570714ff7d75f35 = function(arg0) {
+    imports.wbg.__wbg_value_0c248a78fdc8e19f = function(arg0) {
         const ret = getObject(arg0).value;
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_iterator_7ee1a391d310f8e4 = function() {
+    imports.wbg.__wbg_iterator_db7ca081358d4fb2 = function() {
         const ret = Symbol.iterator;
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_get_2aff440840bb6202 = function() { return handleError(function (arg0, arg1) {
+    imports.wbg.__wbg_get_7b48513de5dc5ea4 = function() { return handleError(function (arg0, arg1) {
         const ret = Reflect.get(getObject(arg0), getObject(arg1));
         return addHeapObject(ret);
     }, arguments) };
-    imports.wbg.__wbg_call_669127b9d730c650 = function() { return handleError(function (arg0, arg1) {
+    imports.wbg.__wbg_call_90c26b09837aba1c = function() { return handleError(function (arg0, arg1) {
         const ret = getObject(arg0).call(getObject(arg1));
         return addHeapObject(ret);
     }, arguments) };
-    imports.wbg.__wbg_isArray_38525be7442aa21e = function(arg0) {
+    imports.wbg.__wbg_isArray_74fb723e24f76012 = function(arg0) {
         const ret = Array.isArray(getObject(arg0));
         return ret;
     };
-    imports.wbg.__wbg_instanceof_ArrayBuffer_c7cc317e5c29cc0d = function(arg0) {
+    imports.wbg.__wbg_instanceof_ArrayBuffer_e7d53d51371448e2 = function(arg0) {
         let result;
         try {
             result = getObject(arg0) instanceof ArrayBuffer;
@@ -628,30 +628,30 @@ function __wbg_get_imports() {
         const ret = result;
         return ret;
     };
-    imports.wbg.__wbg_isSafeInteger_c38b0a16d0c7cef7 = function(arg0) {
+    imports.wbg.__wbg_isSafeInteger_f93fde0dca9820f8 = function(arg0) {
         const ret = Number.isSafeInteger(getObject(arg0));
         return ret;
     };
-    imports.wbg.__wbg_entries_6d727b73ee02b7ce = function(arg0) {
+    imports.wbg.__wbg_entries_9e2e2aa45aa5094a = function(arg0) {
         const ret = Object.entries(getObject(arg0));
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_buffer_344d9b41efe96da7 = function(arg0) {
+    imports.wbg.__wbg_buffer_a448f833075b71ba = function(arg0) {
         const ret = getObject(arg0).buffer;
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_new_d8a000788389a31e = function(arg0) {
+    imports.wbg.__wbg_new_8f67e318f15d7254 = function(arg0) {
         const ret = new Uint8Array(getObject(arg0));
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_set_dcfd613a3420f908 = function(arg0, arg1, arg2) {
+    imports.wbg.__wbg_set_2357bf09366ee480 = function(arg0, arg1, arg2) {
         getObject(arg0).set(getObject(arg1), arg2 >>> 0);
     };
-    imports.wbg.__wbg_length_a5587d6cd79ab197 = function(arg0) {
+    imports.wbg.__wbg_length_1d25fa9e4ac21ce7 = function(arg0) {
         const ret = getObject(arg0).length;
         return ret;
     };
-    imports.wbg.__wbg_instanceof_Uint8Array_19e6f142a5e7e1e1 = function(arg0) {
+    imports.wbg.__wbg_instanceof_Uint8Array_bced6f43aed8c1aa = function(arg0) {
         let result;
         try {
             result = getObject(arg0) instanceof Uint8Array;

backend/src/main.rs

@@ -97,6 +97,15 @@ async fn main() -> anyhow::Result<()> {
             } else if &x == "worker" {
                 tracing::info!("Binary is in 'worker' mode");
                 Mode::Worker
+            } else if &x == "agent" {
+                tracing::info!("Binary is in 'agent' mode");
+                if std::env::var("BASE_INTERNAL_URL").is_err() {
+                    panic!("BASE_INTERNAL_URL is required in agent mode")
+                }
+                if std::env::var("JOB_TOKEN").is_err() {
+                    tracing::warn!("JOB_TOKEN is not passed, hence workers will still create one ephemeral token per job and the DATABASE_URL need to be of a role that can INSERT into the token table")
+                }
+                Mode::Agent
             } else {
                 if &x != "standalone" {
                     tracing::error!("mode not recognized, defaulting to standalone: {x}");
@@ -130,7 +139,7 @@ async fn main() -> anyhow::Result<()> {
         .ok()
         .and_then(|x| x.parse::<bool>().ok())
         .unwrap_or(false)
-        && mode != Mode::Worker;
+        && (mode == Mode::Server || mode == Mode::Standalone);
 
     let server_bind_address: IpAddr = if server_mode {
         std::env::var("SERVER_BIND_ADDR")
@@ -191,33 +200,45 @@ async fn main() -> anyhow::Result<()> {
         None
     };
 
-    let last_mig_version =
-        sqlx::query_scalar!("select version from _sqlx_migrations order by version desc limit 1;")
-            .fetch_optional(&db)
-            .await
-            .ok()
-            .flatten();
+    let is_agent = mode == Mode::Agent;
 
-    tracing::info!(
+    if !is_agent {
+        let last_mig_version = sqlx::query_scalar!(
+            "select version from _sqlx_migrations order by version desc limit 1;"
+        )
+        .fetch_optional(&db)
+        .await
+        .ok()
+        .flatten();
+
+        tracing::info!(
         "Last migration version: {last_mig_version:?}. Starting potential migration of the db if first connection on a new windmill version (can take a while depending on the migration) ...",
     );
 
-    // migration code to avoid break
-    windmill_api::migrate_db(&db).await?;
+        // migration code to avoid break
+        windmill_api::migrate_db(&db).await?;
 
-    let last_mig_version =
-        sqlx::query_scalar!("select version from _sqlx_migrations order by version desc limit 1;")
-            .fetch_optional(&db)
-            .await
-            .ok()
-            .flatten();
+        let new_last_mig_version = sqlx::query_scalar!(
+            "select version from _sqlx_migrations order by version desc limit 1;"
+        )
+        .fetch_optional(&db)
+        .await
+        .ok()
+        .flatten();
 
-    tracing::info!(
-        "Completed potential migration of the db. Last migration version: {last_mig_version:?}",
-    );
-
-    let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());
+        if last_mig_version != new_last_mig_version {
+            tracing::info!(
+                "Completed migration of the db. New  migration version: {}",
+                new_last_mig_version.unwrap_or(-1)
+            );
+        } else {
+            tracing::info!("No migration, db was up-to-date");
+        }
+    }
+    let (killpill_tx, killpill_rx) = tokio::sync::broadcast::channel::<()>(2);
+    let (killpill_phase2_tx, killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2);
+    let shutdown_signal =
+        windmill_common::shutdown_signal(killpill_tx.clone(), killpill_rx.resubscribe());
 
     #[cfg(feature = "enterprise")]
     tracing::info!(
@@ -248,23 +269,29 @@ Windmill Community Edition {GIT_VERSION}
             port_var.unwrap_or(0)
         };
 
+        let default_base_internal_url = format!("http://localhost:{}", port.to_string());
         // since it's only on server mode, the port is statically defined
-        let base_internal_url: String = format!("http://localhost:{}", port.to_string());
+        let base_internal_url: String = if let Ok(base_url) = std::env::var("BASE_INTERNAL_URL") {
+            if !is_agent {
+                tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored unless the mode is 'agent', you can remove it.");
+                default_base_internal_url.clone()
+            } else {
+                base_url
+            }
+        } else {
+            default_base_internal_url.clone()
+        };
 
-        initial_load(&db, tx.clone(), worker_mode, server_mode).await;
+        initial_load(&db, killpill_tx.clone(), worker_mode, server_mode).await;
 
         monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
 
         monitor_pool(&db).await;
 
-        if std::env::var("BASE_INTERNAL_URL").is_ok() {
-            tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
-        }
-
         let addr = SocketAddr::from((server_bind_address, port));
 
         let rsmq2 = rsmq.clone();
-        let (port_tx, port_rx) = tokio::sync::oneshot::channel::<u16>();
+        let (base_internal_tx, base_internal_rx) = tokio::sync::oneshot::channel::<String>();
 
         DirBuilder::new()
             .recursive(true)
@@ -273,43 +300,54 @@ Windmill Community Edition {GIT_VERSION}
             .expect("could not create initial server dir");
 
         let server_f = async {
-            windmill_api::run_server(
-                db.clone(),
-                rsmq2,
-                addr,
-                rx.resubscribe(),
-                port_tx,
-                server_mode,
-            )
-            .await?;
+            if !is_agent {
+                windmill_api::run_server(
+                    db.clone(),
+                    rsmq2,
+                    addr,
+                    killpill_phase2_rx.resubscribe(),
+                    base_internal_tx,
+                    server_mode,
+                )
+                .await?;
+            } else {
+                base_internal_tx
+                    .send(base_internal_url.clone())
+                    .map_err(|e| {
+                        anyhow::anyhow!("Could not send base_internal_url to agent: {e}")
+                    })?;
+            }
             Ok(()) as anyhow::Result<()>
         };
 
         let workers_f = async {
-            let port = port_rx.await?;
-            let base_internal_url: String = format!("http://localhost:{}", port.to_string());
+            let base_internal_url = base_internal_rx.await?;
             if worker_mode {
                 run_workers(
                     db.clone(),
-                    rx.resubscribe(),
-                    tx.clone(),
+                    killpill_rx.resubscribe(),
+                    killpill_tx.clone(),
                     num_workers,
                     base_internal_url.clone(),
                     rsmq.clone(),
                 )
                 .await?;
                 tracing::info!("All workers exited.");
-                tx.send(())?; // signal server to shutdown
+                killpill_tx.send(())?;
+            } else {
+                killpill_rx.resubscribe().recv().await?;
             }
+            tracing::info!("Starting phase 2 of shutdown");
+            killpill_phase2_tx.send(())?;
             Ok(()) as anyhow::Result<()>
         };
 
         let monitor_f = async {
             let db = db.clone();
-            let tx = tx.clone();
+            let tx = killpill_tx.clone();
             let rsmq = rsmq.clone();
 
-            let mut rx = rx.resubscribe();
+            let mut rx = killpill_rx.resubscribe();
             let base_internal_url = base_internal_url.to_string();
             let h = tokio::spawn(async move {
                 let mut listener = retry_listen_pg(&db).await;
@@ -445,8 +483,12 @@ Windmill Community Edition {GIT_VERSION}
                 tracing::error!("Metrics are only available in the EE, ignoring...");
 
                 #[cfg(feature = "enterprise")]
-                windmill_common::serve_metrics(*METRICS_ADDR, rx.resubscribe(), num_workers > 0)
-                    .await;
+                windmill_common::serve_metrics(
+                    *METRICS_ADDR,
+                    killpill_phase2_rx.resubscribe(),
+                    num_workers > 0,
+                )
+                .await;
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -454,7 +496,7 @@ Windmill Community Edition {GIT_VERSION}
         let instance_name = rd_string(8);
         schedule_stats(instance_name, mode, &db, &HTTP_CLIENT).await;
 
-        futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
+        futures::try_join!(shutdown_signal, workers_f, monitor_f, server_f, metrics_f)?;
     } else {
         tracing::info!("Nothing to do, exiting.");
     }

backend/src/monitor.rs

@@ -719,7 +719,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             &db,
             &job.workspace_id,
             &job.permissioned_as,
-            "ephemeral-zombie-jobs",
+            "ephemeral-script",
             *SCRIPT_TOKEN_EXPIRY,
             &job.email,
         )

backend/tests/worker.rs

@@ -124,7 +124,7 @@ impl ApiServer {
 
         let addr = sock.local_addr().unwrap();
         drop(sock);
-        let (port_tx, _port_rx) = tokio::sync::oneshot::channel::<u16>();
+        let (port_tx, _port_rx) = tokio::sync::oneshot::channel::<String>();
 
         let task = tokio::task::spawn(windmill_api::run_server(
             db.clone(),
@@ -961,7 +961,7 @@ fn spawn_test_worker(
     tokio::sync::broadcast::Sender<()>,
     tokio::task::JoinHandle<()>,
 ) {
-    for x in [windmill_worker::LOCK_CACHE_DIR] {
+    for x in [windmill_worker::LOCK_CACHE_DIR, windmill_worker::GO_BIN_CACHE_DIR] {
         std::fs::DirBuilder::new()
             .recursive(true)
             .create(x)
@@ -1084,6 +1084,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1119,6 +1120,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                             mock: None,
                             timeout: None,
                             priority: None,
+                            delete_after_use: None,
                         }],
                     },
                     stop_after_if: Default::default(),
@@ -1130,6 +1132,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
             ],
             same_worker: false,
@@ -1230,6 +1233,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1276,6 +1280,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 mock: None,
                                 timeout: None,
                                 priority: None,
+                                delete_after_use: None,
                             },
                             FlowModule {
                                 id: "e".to_string(),
@@ -1308,6 +1313,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 mock: None,
                                 timeout: None,
                                 priority: None,
+                                delete_after_use: None
                             },
                         ],
                     },
@@ -1320,26 +1326,26 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None
                 },
                 FlowModule {
                     id: "c".to_string(),
                     value: FlowModuleValue::RawScript {
                         input_transforms: [
-
-                        (
-                            "loops".to_string(),
-                            InputTransform::Javascript { expr: "results.b".to_string() },
-                        ),
-                        (
-                            "path".to_string(),
-                            InputTransform::Static { value: json!("outer.txt") },
-                        ),
-                        (
-                            "path2".to_string(),
-                            InputTransform::Static { value: json!("inner.txt") },
-                        ),
-                    ]
-                    .into(),
+                            (
+                                "loops".to_string(),
+                                InputTransform::Javascript { expr: "results.b".to_string() },
+                            ),
+                            (
+                                "path".to_string(),
+                                InputTransform::Static { value: json!("outer.txt") },
+                            ),
+                            (
+                                "path2".to_string(),
+                                InputTransform::Static { value: json!("inner.txt") },
+                            ),
+                        ]
+                        .into(),
                         language: ScriptLang::Deno,
                         content: r#"export async function main(path: string, loops: string[], path2: string) {
                             return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`);
@@ -1360,6 +1366,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
             ],
             same_worker: true,
@@ -2579,8 +2586,10 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
                     ws_error_handler_muted: None,
                     priority: None,
                     dedicated_worker: None,
+                    timeout: None,
                 },
                 draft_only: None,
+                deployment_message: None,
             },
         )
         .await
@@ -2802,10 +2811,9 @@ async fn test_complex_flow_restart(db: Pool<Postgres>) {
         }),
     }).run_until_complete(&db, port).await;
 
-    assert_eq!(
-        first_run_result.json_result().unwrap(),
-        restarted_flow_result.json_result().unwrap()
-    );
+    let first_run_result_int = serde_json::from_value::<i32>(first_run_result.json_result().unwrap()).expect("first_run_result was not an int");
+    let restarted_flow_result_int = serde_json::from_value::<i32>(restarted_flow_result.json_result().unwrap()).expect("restarted_flow_result was not an int");
+    assert_eq!(first_run_result_int, restarted_flow_result_int);
 }
 
 #[sqlx::test(fixtures("base"))]
@@ -3129,6 +3137,10 @@ async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: Stri
             schema: std::collections::HashMap::new(),
             ws_error_handler_muted: Some(false),
             priority: None,
+            delete_after_use: None,
+            timeout: None,
+            restart_unless_cancelled: None,
+            deployment_message: None,
         },
     ).await.unwrap();
 

backend/windmill-api/Cargo.toml

@@ -9,7 +9,8 @@ name = "windmill_api"
 path = "src/lib.rs"
 
 [features]
-enterprise = ["windmill-queue/enterprise", "async-stripe", "samael", "windmill-audit/enterprise"]
+enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise"]
+saml = ["samael"]
 benchmark = []
 
 [dependencies]
@@ -82,5 +83,8 @@ tokenizers.workspace = true
 candle-core.workspace = true
 candle-transformers.workspace = true
 candle-nn.workspace = true
-aws-sdk-s3 = "0.36.0"
-aws-config = "0.57.2"
+aws-sdk-s3 = "0.39.1"
+aws-config = "1.0.0"
+polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] }
+polars-io = { version = "0.35.2", features = ["csv"] }
+object_store = { version = "0.8.0", features = ["aws"] }

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.210.1
+  version: 1.224.0
   title: Windmill API
 
   contact:
@@ -1192,6 +1192,8 @@ paths:
                     type: boolean
                   large_file_storage:
                     $ref: "#/components/schemas/LargeFileStorage"
+                  git_sync:
+                    $ref: "#/components/schemas/WorkspaceGitSync"
                 required:
                   - code_completion_enabled
 
@@ -1513,6 +1515,32 @@ paths:
             application/json:
               schema: {}
 
+  /w/{workspace}/workspaces/edit_git_sync_config:
+    post:
+      summary: edit workspace git sync settings
+      operationId: editWorkspaceGitSyncConfig
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: Workspace Git sync settings
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                git_sync_settings:
+                  $ref: "#/components/schemas/WorkspaceGitSync"
+
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/workspaces/get_large_file_storage_config:
     get:
       summary: get large file storage config
@@ -1634,6 +1662,11 @@ paths:
       operationId: listTokens
       tags:
         - user
+      parameters:
+        - name: exclude_ephemeral
+          in: query
+          schema:
+            type: boolean
       responses:
         "200":
           description: truncated token
@@ -2192,8 +2225,7 @@ paths:
 
   /w/{workspace}/resources/get_value_interpolated/{path}:
     get:
-      summary:
-        get resource interpolated (variables and resources are fully unrolled)
+      summary: get resource interpolated (variables and resources are fully unrolled)
       operationId: getResourceValueInterpolated
       tags:
         - resource
@@ -2881,8 +2913,7 @@ paths:
           schema:
             type: string
         - name: first_parent_hash
-          description:
-            mask to filter scripts whom first direct parent has exact hash
+          description: mask to filter scripts whom first direct parent has exact hash
           in: query
           schema:
             type: string
@@ -3077,8 +3108,7 @@ paths:
 
   /workers/custom_tags:
     get:
-      summary:
-        get all instance custom tags (tags are used to dispatch jobs to
+      summary: get all instance custom tags (tags are used to dispatch jobs to
         different worker groups)
       operationId: getCustomTags
       tags:
@@ -3129,8 +3159,7 @@ paths:
 
   /w/{workspace}/scripts/delete/h/{hash}:
     post:
-      summary:
-        delete script by hash (erase content but keep hash, require admin)
+      summary: delete script by hash (erase content but keep hash, require admin)
       operationId: deleteScriptByHash
       tags:
         - script
@@ -3196,6 +3225,53 @@ paths:
               schema:
                 $ref: "#/components/schemas/NewScriptWithDraft"
 
+  /w/{workspace}/scripts/history/p/{path}:
+    get:
+      summary: get history of a script by path
+      operationId: getScriptHistoryByPath
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script history
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ScriptHistory"
+
+  /w/{workspace}/scripts/history_update/h/{hash}/p/{path}:
+    post:
+      summary: update history of a script
+      operationId: updateScriptHistory
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptHash"
+        - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: Script deployment message
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                deployment_msg:
+                  type: string
+      responses:
+        "200":
+          description: success
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/scripts/raw/p/{path}:
     get:
       summary: raw script by path
@@ -3322,16 +3398,14 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/NewJobId"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -3713,6 +3787,8 @@ paths:
                   properties:
                     draft_only:
                       type: boolean
+                    deployment_message:
+                      type: string
       responses:
         "201":
           description: flow created
@@ -3736,7 +3812,12 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/OpenFlowWPath"
+              allOf:
+                - $ref: "#/components/schemas/OpenFlowWPath"
+                - type: object
+                  properties:
+                    deployment_message:
+                      type: string
 
       responses:
         "200":
@@ -3974,6 +4055,8 @@ paths:
                   $ref: "#/components/schemas/Policy"
                 draft_only:
                   type: boolean
+                deployment_message:
+                  type: string
               required:
                 - path
                 - value
@@ -4038,6 +4121,53 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersionWDraft"
 
+  /w/{workspace}/apps/history/p/{path}:
+    get:
+      summary: get app history by path
+      operationId: getAppHistoryByPath
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app history
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/AppHistory"
+
+  /w/{workspace}/apps/history_update/a/{id}/v/{version}:
+    post:
+      summary: update app history
+      operationId: updateAppHistory
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/PathId"
+        - $ref: "#/components/parameters/PathVersion"
+      requestBody:
+        description: App deployment message
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                deployment_msg:
+                  type: string
+      responses:
+        "200":
+          description: success
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/apps_u/public_app/{path}:
     get:
       summary: get public app by secret
@@ -4228,6 +4358,8 @@ paths:
                 value: {}
                 policy:
                   $ref: "#/components/schemas/Policy"
+                deployment_message:
+                  type: string
       responses:
         "200":
           description: app updated
@@ -4304,8 +4436,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4313,8 +4444,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
           in: query
           schema:
             type: boolean
@@ -4366,8 +4496,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4375,8 +4504,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
           in: query
           schema:
             type: boolean
@@ -4414,8 +4542,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4423,8 +4550,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4455,8 +4581,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4489,8 +4614,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4705,8 +4829,7 @@ paths:
         - job
       responses:
         "200":
-          description:
-            the timestamp of the db that can be used to compute the drift
+          description: the timestamp of the db that can be used to compute the drift
           content:
             application/json:
               schema:
@@ -4942,8 +5065,7 @@ paths:
 
   /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
     get:
-      summary:
-        get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
       operationId: getResumeUrls
       tags:
         - job
@@ -5494,8 +5616,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the groups the user is member of (default false)
+          description: only list the groups the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -5683,8 +5804,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the folders the user is member of (default false)
+          description: only list the folders the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -5908,6 +6028,26 @@ paths:
                 items:
                   $ref: "#/components/schemas/WorkerPing"
 
+  /workers/exists_worker_with_tag:
+    get:
+      summary: exists worker with tag
+      operationId: existsWorkerWithTag
+      tags:
+        - worker
+      parameters:
+        - name: tag
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: whether a worker with the tag exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
   /configs/list_worker_groups:
     get:
       summary: list worker groups
@@ -6326,8 +6466,10 @@ paths:
         content:
           application/json:
             schema:
-              s3_resource:
-                $ref: "#/components/schemas/S3Resource"
+              type: object
+              properties:
+                s3_resource:
+                  $ref: "#/components/schemas/S3Resource"
       responses:
         "200":
           description: Connection settings
@@ -6338,6 +6480,152 @@ paths:
                 properties:
                   connection_settings_str:
                     type: string
+  /w/{workspace}/job_helpers/v2/duckdb_connection_settings:
+    post:
+      summary:
+        Converts an S3 resource to the set of instructions necessary to connect
+        DuckDB to an S3 bucket
+      operationId: duckdbConnectionSettingsV2
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: S3 resource path to use to generate the connection settings. If empty, the S3 resource defined in the workspace settings will be used
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                s3_resource_path:
+                  type: string
+      responses:
+        "200":
+          description: Connection settings
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  connection_settings_str:
+                    type: string
+                required:
+                  - connection_settings_str
+
+  /w/{workspace}/job_helpers/polars_connection_settings:
+    post:
+      summary:
+        Converts an S3 resource to the set of arguments necessary to connect
+        Polars to an S3 bucket
+      operationId: polarsConnectionSettings
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: S3 resource to connect to
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                s3_resource:
+                  $ref: "#/components/schemas/S3Resource"
+      responses:
+        "200":
+          description: Connection settings
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  endpoint_url:
+                    type: string
+                  key:
+                    type: string
+                  secret:
+                    type: string
+                  use_ssl:
+                    type: boolean
+                  cache_regions:
+                    type: boolean
+                  client_kwargs:
+                    $ref: "#/components/schemas/PolarsClientKwargs"
+                required:
+                  - endpoint_url
+                  - use_ssl
+                  - cache_regions
+                  - client_kwargs
+  /w/{workspace}/job_helpers/v2/polars_connection_settings:
+    post:
+      summary:
+        Converts an S3 resource to the set of arguments necessary to connect
+        Polars to an S3 bucket
+      operationId: polarsConnectionSettingsV2
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: S3 resource path to use to generate the connection settings. If empty, the S3 resource defined in the workspace settings will be used
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                s3_resource_path:
+                  type: string
+      responses:
+        "200":
+          description: Connection settings
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  s3fs_args:
+                    type: object
+                    properties:
+                      endpoint_url:
+                        type: string
+                      key:
+                        type: string
+                      secret:
+                        type: string
+                      use_ssl:
+                        type: boolean
+                      cache_regions:
+                        type: boolean
+                      client_kwargs:
+                        $ref: "#/components/schemas/PolarsClientKwargs"
+                    required:
+                      - endpoint_url
+                      - use_ssl
+                      - cache_regions
+                      - client_kwargs
+                  polars_cloud_options:
+                    type: object
+                    properties:
+                      aws_endpoint_url:
+                        type: string
+                      aws_access_key_id:
+                        type: string
+                      aws_secret_access_key:
+                        type: string
+                      aws_region:
+                        type: string
+                      aws_allow_http:
+                        type: boolean
+                    required:
+                      - aws_endpoint_url
+                      - aws_region
+                      - aws_allow_http
+                required:
+                  - s3fs_args
+                  - cloud_options
 
   /w/{workspace}/job_helpers/test_connection:
     get:
@@ -6354,26 +6642,105 @@ paths:
             application/json:
               schema: {}
 
-  /w/{workspace}/job_helpers/list_stored_datasets:
-    post:
-      summary: List the dataset keys available in the worspace datasets storage
-      operationId: polarsConnectionSettings
+  /w/{workspace}/job_helpers/list_stored_files:
+    get:
+      summary: List the file keys available in the worspace files storage (S3)
+      operationId: listStoredFiles
       tags:
         - helpers
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
+        - name: max_keys
+          in: query
+          required: true
+          schema:
+            type: integer
+        - name: marker
+          in: query
+          schema:
+            type: string
       responses:
         "200":
-          description: Connection settings
+          description: List of file keys
           content:
             application/json:
               schema:
                 type: object
                 properties:
-                  dataset_keys:
+                  next_marker:
+                    type: string
+                  windmill_large_files:
                     type: array
                     items:
                       $ref: "#/components/schemas/WindmillLargeFile"
+                required:
+                  - windmill_large_files
+
+  /w/{workspace}/job_helpers/load_file_metadata:
+    get:
+      summary: Load metadata of the file
+      operationId: loadFileMetadata
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: file_key
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: FileMetadata
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WindmillFileMetadata"
+
+  /w/{workspace}/job_helpers/load_file_preview:
+    get:
+      summary: Load a preview of the file
+      operationId: loadFilePreview
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: file_key
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: file_size_in_bytes
+          in: query
+          schema:
+            type: integer
+        - name: file_mime_type
+          in: query
+          schema:
+            type: string
+        - name: csv_separator
+          in: query
+          schema:
+            type: string
+        - name: csv_has_header
+          in: query
+          schema:
+            type: boolean
+        - name: read_bytes_from
+          in: query
+          schema:
+            type: integer
+        - name: read_bytes_length
+          in: query
+          schema:
+            type: integer
+      responses:
+        "200":
+          description: FilePreview
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WindmillFilePreview"
 
 components:
   securitySchemes:
@@ -6453,6 +6820,12 @@ components:
       required: true
       schema:
         type: integer
+    PathVersion:
+      name: version
+      in: path
+      required: true
+      schema:
+        type: integer
     Name:
       name: name
       in: path
@@ -6467,8 +6840,7 @@ components:
         type: integer
     PerPage:
       name: per_page
-      description:
-        number of items to return for a given page (default 30, max 100)
+      description: number of items to return for a given page (default 30, max 100)
       in: query
       schema:
         type: integer
@@ -6602,8 +6974,7 @@ components:
         type: boolean
     ArgsFilter:
       name: args
-      description:
-        filter on jobs containing those args as a json subset (@> in postgres)
+      description: filter on jobs containing those args as a json subset (@> in postgres)
       in: query
       schema:
         type: string
@@ -6615,8 +6986,7 @@ components:
         type: string
     ResultFilter:
       name: result
-      description:
-        filter on jobs containing those result as a json subset (@> in postgres)
+      description: filter on jobs containing those result as a json subset (@> in postgres)
       in: query
       schema:
         type: string
@@ -6785,6 +7155,12 @@ components:
           type: boolean
         priority:
           type: integer
+        restart_unless_cancelled:
+          type: boolean
+        timeout:
+          type: integer
+        delete_after_use:
+          type: boolean
       required:
         - hash
         - path
@@ -6863,6 +7239,14 @@ components:
           type: boolean
         priority:
           type: integer
+        restart_unless_cancelled:
+          type: boolean
+        timeout:
+          type: integer
+        delete_after_use:
+          type: boolean
+        deployment_message:
+          type: string
       required:
         - path
         - summary
@@ -6882,6 +7266,16 @@ components:
           required:
             - hash
 
+    ScriptHistory:
+      type: object
+      properties:
+        script_hash:
+          type: string
+        deployment_msg:
+          type: string
+      required:
+        - script_hash
+
     ScriptArgs:
       type: object
       additionalProperties: {}
@@ -6998,6 +7392,7 @@ components:
               "flowpreview",
               "script_hub",
               "identity",
+              "deploymentcallback",
             ]
         schedule_path:
           type: string
@@ -7106,6 +7501,7 @@ components:
               "flowpreview",
               "script_hub",
               "identity",
+              "deploymentcallback",
             ]
         schedule_path:
           type: string
@@ -8130,6 +8526,8 @@ components:
           type: integer
         dedicated_worker:
           type: boolean
+        timeout:
+          type: number
       required:
         - path
         - edited_by
@@ -8152,6 +8550,8 @@ components:
               type: integer
             dedicated_worker:
               type: boolean
+            timeout:
+              type: number
           required:
             - path
 
@@ -8315,6 +8715,16 @@ components:
               type: boolean
             draft: {}
 
+    AppHistory:
+      type: object
+      properties:
+        version:
+          type: integer
+        deployment_msg:
+          type: string
+      required:
+        - version
+
     SlackToken:
       type: object
       properties:
@@ -8358,6 +8768,14 @@ components:
         type: string
         enum: [script, failure, trigger, approval]
 
+    PolarsClientKwargs:
+      type: object
+      properties:
+        region_name:
+          type: string
+      required:
+        - region_name
+
     LargeFileStorage:
       type: object
       properties:
@@ -8372,11 +8790,38 @@ components:
       properties:
         s3:
           type: string
-        s3_bucket:
-          type: string
       required:
         - s3
 
+    WindmillFileMetadata:
+      type: object
+      properties:
+        mime_type:
+          type: string
+        size_in_bytes:
+          type: integer
+        last_modified:
+          type: string
+          format: date-time
+        expires:
+          type: string
+          format: date-time
+        version_id:
+          type: string
+
+    WindmillFilePreview:
+      type: object
+      properties:
+        msg:
+          type: string
+        content:
+          type: string
+        content_type:
+          type: string
+          enum: ["RawText", "Csv", "Parquet", "Unknown"]
+      required:
+        - content_type
+
     S3Resource:
       type: object
       properties:
@@ -8400,3 +8845,14 @@ components:
         - endPoint
         - useSSL
         - pathStyle
+
+    WorkspaceGitSync:
+      type: object
+      properties:
+        script_path:
+          type: string
+        git_repo_resource_path:
+          type: string
+      required:
+        - script_path
+        - git_repo_resource_path

backend/windmill-api/src/apps.rs

@@ -9,6 +9,7 @@ use std::collections::HashMap;
  */
 use crate::{
     db::{ApiAuthed, DB},
+    deployment_metadata_helpers,
     users::{require_owner_of_path, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
@@ -54,6 +55,8 @@ pub fn workspaced_service() -> Router {
         .route("/update/*path", post(update_app))
         .route("/delete/*path", delete(delete_app))
         .route("/create", post(create_app))
+        .route("/history/p/*path", get(get_app_history))
+        .route("/history_update/a/:id/v/:version", post(update_app_history))
 }
 
 pub fn unauthed_service() -> Router {
@@ -124,6 +127,19 @@ pub struct AppWithLastVersionAndDraft {
     pub draft_only: Option<bool>,
 }
 
+#[derive(Serialize)]
+pub struct AppHistory {
+    pub app_id: i64,
+    pub version: i64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub deployment_msg: Option<String>,
+}
+
+#[derive(Deserialize)]
+pub struct AppHistoryUpdate {
+    pub deployment_msg: Option<String>,
+}
+
 pub type StaticFields = HashMap<String, Box<RawValue>>;
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -153,6 +169,7 @@ pub struct CreateApp {
     pub value: serde_json::Value,
     pub policy: Policy,
     pub draft_only: Option<bool>,
+    pub deployment_message: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -161,6 +178,7 @@ pub struct EditApp {
     pub summary: Option<String>,
     pub value: Option<serde_json::Value>,
     pub policy: Option<Policy>,
+    pub deployment_message: Option<String>,
 }
 
 #[derive(Serialize, FromRow)]
@@ -312,6 +330,64 @@ async fn get_app_w_draft(
     Ok(Json(app))
 }
 
+async fn get_app_history(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Vec<AppHistory>> {
+    let mut tx = user_db.begin(&authed).await?;
+    let query_result = sqlx::query!(
+        "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg
+        FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version
+        WHERE a.workspace_id = $1 AND a.path = $2
+        ORDER BY created_at DESC",
+        w_id,
+        path.to_path(),
+    ).fetch_all(&mut *tx).await?;
+    tx.commit().await?;
+
+    let result: Vec<AppHistory> = query_result
+        .into_iter()
+        .map(|row| AppHistory {
+            app_id: row.app_id,
+            version: row.version_id,
+            deployment_msg: row.deployment_msg,
+        })
+        .collect();
+    return Ok(Json(result));
+}
+
+async fn update_app_history(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, app_id, app_version)): Path<(String, i64, i64)>,
+    Json(app_history_update): Json<AppHistoryUpdate>,
+) -> Result<()> {
+    let mut tx = user_db.begin(&authed).await?;
+    let app_path = sqlx::query_scalar!("SELECT path FROM app WHERE id = $1", app_id)
+        .fetch_optional(&mut *tx)
+        .await?;
+
+    if app_path.is_none() {
+        tx.commit().await?;
+        return Err(Error::NotFound(
+            format!("App with ID {app_id} not found").to_string(),
+        ));
+    }
+
+    sqlx::query!(
+        "INSERT INTO deployment_metadata (workspace_id, path, app_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET deployment_msg = $4",
+        w_id,
+        app_path.unwrap(),
+        app_version,
+        app_history_update.deployment_msg,
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+    tx.commit().await?;
+    return Ok(());
+}
+
 async fn get_app_by_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -512,8 +588,8 @@ async fn create_app(
     )
     .await?;
 
-    let tx = PushIsolationLevel::Transaction(tx);
-    let (dependency_job_uuid, tx) = push(
+    let mut tx = PushIsolationLevel::Transaction(tx);
+    let (dependency_job_uuid, new_tx) = push(
         &db,
         tx,
         &w_id,
@@ -538,7 +614,27 @@ async fn create_app(
     )
     .await?;
     tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
-    tx.commit().await?;
+
+    tx = PushIsolationLevel::Transaction(new_tx);
+    tx = deployment_metadata_helpers::handle_deployment_metadata(
+        tx,
+        &authed,
+        &db,
+        &w_id,
+        deployment_metadata_helpers::DeployedObject::App { path: app.path.clone(), version: v_id },
+        app.deployment_message,
+    )
+    .await?;
+
+    match tx {
+        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        _ => {
+            return Err(Error::InternalErr(
+                "Expected a transaction here".to_string(),
+            ));
+        }
+    }
+
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
@@ -753,11 +849,10 @@ async fn update_app(
     )
     .await?;
 
+    let mut tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
+        PushIsolationLevel::Transaction(tx);
     if let Some(v_id) = v_id {
-        let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
-            PushIsolationLevel::Transaction(tx);
-
-        let (dependency_job_uuid, tx) = push(
+        let (dependency_job_uuid, new_tx) = push(
             &db,
             tx,
             &w_id,
@@ -782,10 +877,28 @@ async fn update_app(
         )
         .await?;
         tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
-        tx.commit().await?;
-    } else {
-        tx.commit().await?;
+
+        tx = PushIsolationLevel::Transaction(new_tx);
+        tx = deployment_metadata_helpers::handle_deployment_metadata(
+            tx,
+            &authed,
+            &db,
+            &w_id,
+            deployment_metadata_helpers::DeployedObject::App { path: npath.clone(), version: v_id },
+            ns.deployment_message,
+        )
+        .await?;
     }
+
+    match tx {
+        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        _ => {
+            return Err(Error::InternalErr(
+                "Expected a transaction here".to_string(),
+            ));
+        }
+    }
+
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::UpdateApp {

backend/windmill-api/src/deployment_metadata_helpers.rs

@@ -0,0 +1,176 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use std::collections::HashMap;
+
+use crate::db::ApiAuthed;
+use crate::db::DB;
+use crate::workspaces::{WorkspaceGitRepo, WorkspaceSettings};
+
+use serde_json::json;
+use windmill_common::error::{Error, Result};
+use windmill_common::jobs::JobPayload;
+use windmill_common::scripts::ScriptHash;
+use windmill_common::users::username_to_permissioned_as;
+use windmill_queue::PushIsolationLevel;
+
+#[derive(Clone)]
+pub enum DeployedObject {
+    Script { hash: ScriptHash, path: String },
+    Flow { path: String },
+    App { path: String, version: i64 },
+}
+
+impl DeployedObject {
+    pub fn get_path(&self) -> &str {
+        match self {
+            DeployedObject::Script { path, .. } => path,
+            DeployedObject::Flow { path } => path,
+            DeployedObject::App { path, .. } => path,
+        }
+    }
+}
+
+pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
+    mut tx: PushIsolationLevel<'c, R>,
+    authed: &ApiAuthed,
+    db: &DB,
+    w_id: &str,
+    obj: DeployedObject,
+    deployment_message: Option<String>,
+) -> Result<PushIsolationLevel<'c, R>> {
+    let skip_git_sync = if obj.get_path().starts_with("u/") {
+        tracing::debug!(
+            "Ignoring {} from git sync as it's in a private user folder",
+            obj.get_path()
+        );
+        true
+    } else {
+        false
+    };
+
+    let workspace_git_repo_setting = sqlx::query_as::<_, WorkspaceSettings>(
+        "SELECT * FROM workspace_settings WHERE workspace_id = $1",
+    )
+    .bind(&w_id)
+    .fetch_optional(db)
+    .await?;
+
+    if workspace_git_repo_setting.is_none() {
+        return Err(Error::InternalErr(
+            "No workspace settings found for workspace ID".to_string(),
+        ));
+    }
+
+    let workspace_git_repo = workspace_git_repo_setting
+        .unwrap()
+        .git_sync
+        .map(|conf| serde_json::from_value::<WorkspaceGitRepo>(conf).ok())
+        .flatten();
+
+    let (git_sync_job_uuid, mut new_tx) = if !skip_git_sync && workspace_git_repo.is_some() {
+        let workspace_git_repo = workspace_git_repo.unwrap();
+
+        let mut args: HashMap<String, serde_json::Value> = HashMap::new();
+        args.insert(
+            "repo_url_resource_path".to_string(),
+            json!(workspace_git_repo
+                .git_repo_resource_path
+                .strip_prefix("$res:")),
+        );
+
+        let default_commit_msg: String;
+        match obj.clone() {
+            DeployedObject::Script { path, .. } => {
+                args.insert("path".to_string(), json!(path.to_string()));
+                default_commit_msg = format!("Script '{}' deployed", path);
+            }
+            DeployedObject::Flow { path } => {
+                args.insert("path".to_string(), json!(path.to_string()));
+                default_commit_msg = format!("Flow '{}' deployed", path);
+            }
+            DeployedObject::App { path, .. } => {
+                args.insert("path".to_string(), json!(path.to_string()));
+                default_commit_msg = format!("App '{}' deployed", path);
+            }
+        }
+        args.insert(
+            "commit_msg".to_string(),
+            json!(deployment_message.clone().unwrap_or(default_commit_msg)),
+        );
+
+        let (job_uuid, new_tx) = windmill_queue::push(
+            &db,
+            tx,
+            &w_id,
+            JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
+            args,
+            &authed.username,
+            &authed.email,
+            username_to_permissioned_as(&authed.username),
+            None,
+            None,
+            None,
+            None,
+            None,
+            false,
+            false,
+            None,
+            true,
+            None,
+            None,
+            None,
+            None,
+        )
+        .await?;
+        (Some(job_uuid), new_tx)
+    } else {
+        let new_tx = match tx {
+            PushIsolationLevel::Isolated(user_db, authed, rsmq) => {
+                (rsmq, user_db.begin(&authed).await?).into()
+            }
+            PushIsolationLevel::IsolatedRoot(db, rsmq) => (rsmq, db.begin().await?).into(),
+            PushIsolationLevel::Transaction(tx) => tx,
+        };
+        (None, new_tx)
+    };
+
+    // We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
+    let job_uuids = if git_sync_job_uuid.is_some() {
+        vec![git_sync_job_uuid.unwrap()]
+    } else {
+        vec![]
+    };
+    if deployment_message.is_some() || job_uuids.len() > 0 {
+        // if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table
+        match obj.clone() {
+            DeployedObject::Script { path, hash, .. } => {
+                sqlx::query!(
+                    "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+                    w_id, path, hash.0, &job_uuids, deployment_message,
+                )
+            },
+            DeployedObject::Flow { path } => {
+                sqlx::query!(
+                    "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
+                    w_id, path, &job_uuids, deployment_message,
+                )
+            }
+            DeployedObject::App { path, version } => {
+                sqlx::query!(
+                    "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+                    w_id, path, version, &job_uuids, deployment_message,
+                )
+            }
+        }.execute(&mut new_tx)
+        .await?;
+    }
+
+    tx = PushIsolationLevel::Transaction(new_tx);
+    return Ok(tx);
+}

backend/windmill-api/src/embeddings.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, path::PathBuf, sync::Arc};
 
-use anyhow::{self, Error, Result};
+use anyhow::{anyhow, Error, Result};
 use axum::{
     extract::{Path, Query},
     routing::get,
@@ -228,14 +228,31 @@ impl EmbeddingsDb {
         self.db
             .create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
 
-        let response = http_get_from_hub(
-            &HTTP_CLIENT,
-            "https://hub.windmill.dev/scripts/embeddings",
-            false,
-            None,
-            pg_db,
-        )
-        .await?;
+        let response = HTTP_CLIENT
+            .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
+            .send()
+            .await;
+        let response =
+            if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
+                tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
+                http_get_from_hub(
+                    &HTTP_CLIENT,
+                    "https://hub.windmill.dev/scripts/embeddings",
+                    false,
+                    None,
+                    pg_db,
+                )
+                .await?
+            } else {
+                response.unwrap()
+            };
+        if response.error_for_status_ref().is_err() {
+            return Err(anyhow!(
+                "Failed to get scripts embeddings from hub with error code: {}",
+                response.status()
+            ));
+        }
+
         let hub_scripts = response.json::<Vec<HubScript>>().await?;
 
         for script in &hub_scripts {
@@ -257,14 +274,31 @@ impl EmbeddingsDb {
             self.db.insert_into_collection("scripts", embedding)?;
         }
 
-        let response = http_get_from_hub(
-            &HTTP_CLIENT,
-            "https://hub.windmill.dev/resource_types/embeddings",
-            false,
-            None,
-            pg_db,
-        )
-        .await?;
+        let response = HTTP_CLIENT
+            .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
+            .send()
+            .await;
+        let response = if response.is_err()
+            || response.as_ref().unwrap().error_for_status_ref().is_err()
+        {
+            tracing::warn!("Failed to get resource types embeddings from bucket, trying hub...");
+            http_get_from_hub(
+                &HTTP_CLIENT,
+                "https://hub.windmill.dev/resource_types/embeddings",
+                false,
+                None,
+                pg_db,
+            )
+            .await?
+        } else {
+            response.unwrap()
+        };
+        if response.error_for_status_ref().is_err() {
+            return Err(anyhow!(
+                "Failed to get resource types embeddings from hub with error code: {}",
+                response.status()
+            ));
+        }
         let hub_resource_types = response.json::<Vec<HubResourceType>>().await?;
 
         let resource_types: Vec<ResourceType> =

backend/windmill-api/src/flows.rs

@@ -7,6 +7,7 @@
  */
 
 use crate::db::ApiAuthed;
+use crate::deployment_metadata_helpers;
 use crate::{
     db::DB,
     schedule::clear_schedule,
@@ -354,8 +355,8 @@ async fn create_flow(
     )
     .await?;
 
-    let tx = PushIsolationLevel::Transaction(tx);
-    let (dependency_job_uuid, mut tx) = push(
+    let mut tx = PushIsolationLevel::Transaction(tx);
+    let (dependency_job_uuid, mut new_tx) = push(
         &db,
         tx,
         &w_id,
@@ -389,9 +390,28 @@ async fn create_flow(
         nf.path,
         w_id
     )
-    .execute(&mut tx)
+    .execute(&mut new_tx)
     .await?;
-    tx.commit().await?;
+
+    tx = PushIsolationLevel::Transaction(new_tx);
+    tx = deployment_metadata_helpers::handle_deployment_metadata(
+        tx,
+        &authed,
+        &db,
+        &w_id,
+        deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
+        nf.deployment_message,
+    )
+    .await?;
+
+    match tx {
+        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        _ => {
+            return Err(Error::InternalErr(
+                "Expected a transaction here".to_string(),
+            ));
+        }
+    }
 
     webhook.send_message(
         w_id.clone(),
@@ -520,6 +540,7 @@ async fn update_flow(
     .await?;
 
     if let Some(schedule) = schedule {
+        clear_schedule(tx.transaction_mut(), &flow_path, &w_id).await?;
         schedulables.push(schedule);
     }
 
@@ -564,9 +585,9 @@ async fn update_flow(
         },
     );
 
-    let tx = PushIsolationLevel::Transaction(tx);
+    let mut tx = PushIsolationLevel::Transaction(tx);
 
-    let (dependency_job_uuid, mut tx) = push(
+    let (dependency_job_uuid, mut new_tx) = push(
         &db,
         tx,
         &w_id,
@@ -599,17 +620,37 @@ async fn update_flow(
         nf.path,
         w_id
     )
-    .execute(&mut tx)
+    .execute(&mut new_tx)
     .await?;
     if let Some(old_dep_job) = old_dep_job {
         sqlx::query!(
             "UPDATE queue SET canceled = true WHERE id = $1",
             old_dep_job
         )
-        .execute(&mut tx)
+        .execute(&mut new_tx)
         .await?;
     }
-    tx.commit().await?;
+
+    tx = PushIsolationLevel::Transaction(new_tx);
+    tx = deployment_metadata_helpers::handle_deployment_metadata(
+        tx,
+        &authed,
+        &db,
+        &w_id,
+        deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
+        nf.deployment_message,
+    )
+    .await?;
+
+    match tx {
+        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        _ => {
+            return Err(Error::InternalErr(
+                "Expected a transaction here".to_string(),
+            ));
+        }
+    }
+
     Ok(nf.path.to_string())
 }
 
@@ -822,6 +863,7 @@ mod tests {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -847,6 +889,7 @@ mod tests {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -869,6 +912,7 @@ mod tests {
                     mock: None,
                     timeout: None,
                     priority: None,
+                    delete_after_use: None,
                 },
             ],
             failure_module: Some(FlowModule {
@@ -890,14 +934,15 @@ mod tests {
                 mock: None,
                 timeout: None,
                 priority: None,
+                delete_after_use: None,
             }),
             same_worker: false,
             concurrent_limit: None,
             concurrency_time_window_s: None,
             skip_expr: None,
             cache_ttl: None,
-            ws_error_handler_muted: None,
             priority: None,
+            early_return: None,
         };
         let expect = serde_json::json!({
           "modules": [

backend/windmill-api/src/job_helpers.rs

@@ -1,13 +1,28 @@
+use std::cmp;
+
 use crate::{
-    db::DB, resources::transform_json_value, users::Tokened, workspaces::LargeFileStorage,
+    db::DB, resources::get_resource_value_interpolated_internal, users::Tokened,
+    workspaces::LargeFileStorage,
 };
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
 use axum::{
-    extract::Path,
+    extract::{Path, Query},
     routing::{get, post},
     Extension, Json, Router,
 };
 use hyper::http;
+use object_store::ClientConfigKey;
+use polars::{
+    io::{
+        cloud::{AmazonS3ConfigKey, CloudOptions},
+        SerReader,
+    },
+    lazy::{
+        dsl::col,
+        frame::{LazyFrame, ScanArgsParquet},
+    },
+    prelude::CsvReader,
+};
 use serde::{Deserialize, Serialize};
 use tower_http::cors::{Any, CorsLayer};
 use windmill_common::{db::UserDB, error};
@@ -25,14 +40,34 @@ pub fn workspaced_service() -> Router {
             "/duckdb_connection_settings",
             post(duckdb_connection_settings).layer(cors.clone()),
         )
+        .route(
+            "/v2/duckdb_connection_settings",
+            post(duckdb_connection_settings_v2).layer(cors.clone()),
+        )
         .route(
             "/polars_connection_settings",
             post(polars_connection_settings).layer(cors.clone()),
         )
+        .route(
+            "/v2/polars_connection_settings",
+            post(polars_connection_settings_v2).layer(cors.clone()),
+        )
+        .route(
+            "/blah",
+            post(polars_connection_settings_v2).layer(cors.clone()),
+        )
         .route("/test_connection", get(test_connection).layer(cors.clone()))
         .route(
-            "/list_stored_datasets",
-            get(list_stored_datasets).layer(cors.clone()),
+            "/list_stored_files",
+            get(list_stored_files).layer(cors.clone()),
+        )
+        .route(
+            "/load_file_metadata",
+            get(load_file_metadata).layer(cors.clone()),
+        )
+        .route(
+            "/load_file_preview",
+            get(load_file_preview).layer(cors.clone()),
         )
 }
 
@@ -91,13 +126,83 @@ async fn duckdb_connection_settings(
     return Ok(Json(response));
 }
 
+#[derive(Deserialize)]
+struct DuckdbConnectionSettingsQueryV2 {
+    s3_resource_path: Option<String>,
+}
+
+async fn duckdb_connection_settings_v2(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Json(query): Json<DuckdbConnectionSettingsQueryV2>,
+) -> error::JsonResult<DuckdbConnectionSettingsResponse> {
+    let s3_resource_opt = match query.s3_resource_path {
+        Some(s3_resource_path) => {
+            get_s3_resource(
+                &authed,
+                &user_db,
+                &db,
+                &token,
+                &w_id,
+                s3_resource_path.as_str(),
+            )
+            .await?
+        }
+        None => get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?,
+    };
+    let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
+        "No datasets storage resource defined at the workspace level".to_string(),
+    ))?;
+    return duckdb_connection_settings(
+        Path(w_id),
+        Json(DuckdbConnectionSettingsQuery { s3_resource }),
+    )
+    .await;
+}
+
 #[derive(Deserialize)]
 struct PolarsConnectionSettingsQuery {
     s3_resource: S3Resource,
 }
 
+#[derive(Serialize)]
+struct PolarsConnectionSettings {
+    pub region_name: String,
+}
+
+async fn polars_connection_settings(
+    Path(_w_id): Path<String>,
+    Json(query): Json<PolarsConnectionSettingsQuery>,
+) -> error::JsonResult<S3fsArgs> {
+    let s3_resource = query.s3_resource;
+
+    let response = S3fsArgs {
+        endpoint_url: render_endpoint(&s3_resource),
+        key: s3_resource.access_key,
+        secret: s3_resource.secret_key,
+        use_ssl: s3_resource.use_ssl,
+        cache_regions: false,
+        client_kwargs: PolarsConnectionSettings { region_name: s3_resource.region },
+    };
+    return Ok(Json(response));
+}
+
+#[derive(Deserialize)]
+struct PolarsConnectionSettingsQueryV2 {
+    s3_resource_path: Option<String>,
+}
+
 #[derive(Serialize)]
 struct PolarsConnectionSettingsResponse {
+    s3fs_args: S3fsArgs,
+    polars_cloud_options: PolarsCloudOptions,
+}
+
+#[derive(Serialize)]
+struct S3fsArgs {
     endpoint_url: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     key: Option<String>,
@@ -109,37 +214,63 @@ struct PolarsConnectionSettingsResponse {
 }
 
 #[derive(Serialize)]
-struct PolarsConnectionSettings {
-    pub region_name: String,
+struct PolarsCloudOptions {
+    aws_endpoint_url: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    aws_access_key_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    aws_secret_access_key: Option<String>,
+    aws_region: String,
+    aws_allow_http: bool,
 }
 
-async fn polars_connection_settings(
-    Path(_w_id): Path<String>,
-    Json(query): Json<PolarsConnectionSettingsQuery>,
+async fn polars_connection_settings_v2(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Json(query): Json<PolarsConnectionSettingsQueryV2>,
 ) -> error::JsonResult<PolarsConnectionSettingsResponse> {
-    let s3_resource = query.s3_resource;
-
-    let response = PolarsConnectionSettingsResponse {
-        endpoint_url: s3_resource.endpoint,
-        key: s3_resource.access_key,
-        secret: s3_resource.secret_key,
-        use_ssl: s3_resource.use_ssl,
-        cache_regions: false,
-        client_kwargs: PolarsConnectionSettings { region_name: s3_resource.region },
+    let s3_resource_opt = match query.s3_resource_path {
+        Some(s3_resource_path) => {
+            get_s3_resource(
+                &authed,
+                &user_db,
+                &db,
+                &token,
+                &w_id,
+                s3_resource_path.as_str(),
+            )
+            .await?
+        }
+        None => get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?,
+    };
+    let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
+        "No datasets storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3fs = polars_connection_settings(
+        Path(w_id),
+        Json(PolarsConnectionSettingsQuery { s3_resource: s3_resource.clone() }),
+    )
+    .await?
+    .0;
+    let response = PolarsConnectionSettingsResponse {
+        s3fs_args: s3fs,
+        polars_cloud_options: PolarsCloudOptions {
+            aws_endpoint_url: render_endpoint(&s3_resource),
+            aws_access_key_id: s3_resource.access_key,
+            aws_secret_access_key: s3_resource.secret_key,
+            aws_region: s3_resource.region,
+            aws_allow_http: !s3_resource.use_ssl,
+        },
     };
-
     return Ok(Json(response));
 }
 
-#[derive(Serialize)]
-struct ListStoredDatasetsResponse {
-    windmill_large_files: Vec<WindmillLargeFile>,
-}
-
-#[derive(Serialize, Clone)]
+#[derive(Serialize, Deserialize, Clone)]
 struct WindmillLargeFile {
     s3: String,
-    s3_bucket: Option<String>,
 }
 
 async fn test_connection(
@@ -156,7 +287,9 @@ async fn test_connection(
         ));
     }
 
-    let s3_resource = s3_resource_opt.unwrap();
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
     let s3_client = build_s3_client(&s3_resource);
     s3_client
         .list_objects()
@@ -168,62 +301,316 @@ async fn test_connection(
     return Ok(Json(()));
 }
 
-async fn list_stored_datasets(
+#[derive(Deserialize)]
+struct ListStoredFilesQuery {
+    pub max_keys: i32,
+    pub marker: Option<String>,
+}
+
+#[derive(Serialize)]
+struct ListStoredDatasetsResponse {
+    windmill_large_files: Vec<WindmillLargeFile>,
+    pub next_marker: Option<String>,
+}
+
+async fn list_stored_files(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
     Tokened { token }: Tokened,
     Path(w_id): Path<String>,
+    Query(query): Query<ListStoredFilesQuery>,
 ) -> error::JsonResult<ListStoredDatasetsResponse> {
     let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
 
-    let s3_resource = s3_resource_opt.unwrap();
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
     let s3_client = build_s3_client(&s3_resource);
 
-    let mut stored_datasets = Vec::<WindmillLargeFile>::new();
     let s3_bucket = s3_resource.bucket;
-    loop {
-        let mut list_object_query = s3_client
-            .list_objects()
-            .bucket(s3_bucket.clone())
-            .max_keys(32);
 
-        if let Some(last_dataset_name) = stored_datasets.last() {
-            // if stored_datasets already has some elements, it means we're looping through pages
-            // according to AWS SDK docs, we can set the marker to the last returned dataset
-            list_object_query = list_object_query.set_marker(Some(last_dataset_name.clone().s3))
-        }
+    let list_object_query = s3_client
+        .list_objects()
+        .bucket(s3_bucket.clone())
+        .max_keys(query.max_keys)
+        .set_marker(query.marker);
 
-        let bucket_objects = list_object_query
-            .send()
-            .await
-            .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+    let bucket_objects = list_object_query
+        .send()
+        .await
+        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
 
-        let page_datasets = bucket_objects
-            .contents()
-            .iter()
-            .filter(|object| object.key().is_some())
-            .map(|object| object.key())
-            .map(Option::unwrap)
-            .map(&str::to_string)
-            .map(|object_key| WindmillLargeFile {
-                s3: object_key.clone(),
-                s3_bucket: Some(s3_bucket.clone()),
-            })
-            .collect::<Vec<WindmillLargeFile>>();
+    let stored_datasets = bucket_objects
+        .contents()
+        .iter()
+        .filter(|object| object.key().is_some())
+        .map(|object| object.key())
+        .map(Option::unwrap)
+        .map(&str::to_string)
+        .map(|object_key| WindmillLargeFile { s3: object_key.clone() })
+        .collect::<Vec<WindmillLargeFile>>();
 
-        stored_datasets.extend(page_datasets.clone());
-
-        if !bucket_objects.is_truncated() {
-            break;
-        }
+    #[cfg(not(feature = "enterprise"))]
+    if stored_datasets.len() > 20 {
+        return Err(error::Error::ExecutionErr(
+            "The workspace s3 bucket contains more than 20 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature."
+                .to_string(),
+        ));
     }
 
+    let next_marker = if bucket_objects.is_truncated().unwrap_or(false) {
+        if bucket_objects.next_marker().is_some() {
+            // some S3 providers returns the next marker for us. If that's the case just re-use it
+            bucket_objects.next_marker().map(|v| v.to_owned())
+        } else {
+            // others, like AWS, doesn't and implicitly expect users to return the last key of the current page
+            stored_datasets.last().map(|v| v.s3.clone())
+        }
+    } else {
+        None
+    };
+
     return Ok(Json(ListStoredDatasetsResponse {
         windmill_large_files: stored_datasets,
+        next_marker: next_marker,
     }));
 }
 
+#[derive(Deserialize)]
+struct LoadFileMetadataQuery {
+    pub file_key: String,
+}
+
+#[derive(Serialize)]
+struct LoadFileMetadataResponse {
+    pub mime_type: Option<String>,
+    pub size_in_bytes: Option<i64>,
+    pub last_modified: Option<chrono::DateTime<chrono::Utc>>,
+    pub expires: Option<chrono::DateTime<chrono::Utc>>,
+    pub version_id: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct LoadFilePreviewQuery {
+    pub file_key: String,
+
+    // The two options below are requested from s3 with an additional query is not set
+    pub file_size_in_bytes: Option<i64>,
+    pub file_mime_type: Option<String>,
+
+    // For CSV files, the separator needs to be specified
+    pub csv_separator: Option<String>, // defaults to ','
+    pub csv_has_header: Option<bool>,  // defaults to true
+
+    // Specify the content length to be read. Both will be taken into account when reading files, except for:
+    // - CSVs: only the length will be taken into account, a CSV file larger than this will be truncated.
+    //   Note that truncated CSV files might not be valid CSV files anymore, and therefore the preview might fail
+    // - Parquet files: Parquet files are lazy-loaded. Therefore none of those params will be taken into account
+    pub read_bytes_from: i64,
+    pub read_bytes_length: i64,
+}
+
+#[derive(Serialize)]
+struct LoadFilePreviewResponse {
+    pub content: Option<String>,
+    pub content_type: WindmillContentType,
+    pub msg: Option<String>,
+}
+
+#[derive(Serialize)]
+enum WindmillContentType {
+    RawText,
+    Csv,
+    Parquet,
+    Unknown,
+}
+
+async fn load_file_metadata(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Query(query): Query<LoadFileMetadataQuery>,
+) -> error::JsonResult<LoadFileMetadataResponse> {
+    let file_key = query.file_key.clone();
+    let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
+
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3_client = build_s3_client(&s3_resource);
+
+    let s3_bucket = s3_resource.bucket.clone();
+    let s3_object_metadata = s3_client
+        .head_object()
+        .bucket(&s3_bucket)
+        .key(&file_key)
+        .send()
+        .await
+        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+
+    let response = LoadFileMetadataResponse {
+        mime_type: s3_object_metadata.content_type().map(&str::to_string),
+        size_in_bytes: s3_object_metadata.content_length(),
+        last_modified: s3_object_metadata
+            .last_modified()
+            .map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos()))
+            .flatten(),
+        expires: s3_object_metadata
+            .expires()
+            .map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos()))
+            .flatten(),
+        version_id: s3_object_metadata.version_id().map(&str::to_string),
+    };
+    return Ok(Json(response));
+}
+
+async fn load_file_preview(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Query(query): Query<LoadFilePreviewQuery>,
+) -> error::JsonResult<LoadFilePreviewResponse> {
+    // query validation
+    if query.read_bytes_length > 8 * 1024 * 1024 {
+        return Err(error::Error::BadRequest(
+            "Cannot load file bigger than 8MB".to_string(),
+        ));
+    }
+
+    let file_key = query.file_key.clone();
+    let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
+
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3_client = build_s3_client(&s3_resource);
+
+    let s3_bucket = s3_resource.bucket.clone();
+
+    // if content length is provided in the request, use it, otherwise get it from s3
+    let (s3_object_mime_type, s3_object_content_length) =
+        if query.file_size_in_bytes.is_some() || query.file_mime_type.is_some() {
+            (query.file_mime_type.clone(), query.file_size_in_bytes)
+        } else {
+            let s3_object_metadata = s3_client
+                .head_object()
+                .bucket(&s3_bucket)
+                .key(&file_key)
+                .send()
+                .await
+                .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+            (
+                s3_object_metadata.content_type().map(|v| v.to_owned()),
+                s3_object_metadata.content_length(),
+            )
+        };
+
+    let file_chunk_length = if s3_object_content_length.is_some() {
+        cmp::min(
+            query.read_bytes_length,
+            s3_object_content_length.unwrap() - query.read_bytes_from,
+        )
+    } else {
+        query.read_bytes_length
+    };
+    let content_type: WindmillContentType;
+    let content_preview = match s3_object_mime_type.as_deref() {
+        Some("text/csv") => {
+            content_type = WindmillContentType::Csv;
+            csv_file_preview_with_fallback(
+                &s3_client,
+                &s3_bucket,
+                &file_key,
+                file_chunk_length,
+                query.csv_separator,
+                query.csv_has_header,
+            )
+            .await
+        }
+        Some(mt)
+            if mt.starts_with("text/")
+                || mt == "application/json"
+                || mt == "application/x-yaml" =>
+        {
+            content_type = WindmillContentType::RawText;
+            read_s3_text_object_head(
+                &s3_client,
+                &s3_bucket,
+                &file_key,
+                query.read_bytes_from,
+                file_chunk_length,
+            )
+            .await
+        }
+        mt_opt => {
+            let lowercased_file_key = file_key.to_lowercase();
+            // sometimes S3 doesn't infer the content type on upload. Guess it from the file extension
+            if lowercased_file_key.ends_with(".parquet") {
+                content_type = WindmillContentType::Parquet;
+                read_s3_parquet_object_head(&s3_resource, &file_key).await
+            } else if lowercased_file_key.ends_with(".csv") {
+                content_type = WindmillContentType::Csv;
+                csv_file_preview_with_fallback(
+                    &s3_client,
+                    &s3_bucket,
+                    &file_key,
+                    file_chunk_length,
+                    query.csv_separator,
+                    query.csv_has_header,
+                )
+                .await
+            } else if lowercased_file_key.ends_with(".json")
+                || lowercased_file_key.ends_with(".yaml")
+                || lowercased_file_key.ends_with(".yml")
+                || lowercased_file_key.ends_with(".xml")
+                || lowercased_file_key.ends_with(".txt")
+                || lowercased_file_key.ends_with(".log")
+            {
+                content_type = WindmillContentType::RawText;
+                read_s3_text_object_head(
+                    &s3_client,
+                    &s3_bucket,
+                    &file_key,
+                    query.read_bytes_from,
+                    file_chunk_length,
+                )
+                .await
+            } else {
+                content_type = WindmillContentType::Unknown;
+                let msg = match mt_opt {
+                    Some(mt) => {
+                        format!("Preview is not available for content of type '{}'", mt).to_string()
+                    }
+                    None => "Preview is not available. Content type is unknown or not supported"
+                        .to_string(),
+                };
+                Err(error::Error::ExecutionErr(msg))
+            }
+        }
+    };
+
+    let response: LoadFilePreviewResponse = match content_preview {
+        Ok(content) => LoadFilePreviewResponse {
+            content_type: content_type,
+            content: Some(content),
+            msg: None,
+        },
+
+        Err(err) => LoadFilePreviewResponse {
+            content_type: content_type,
+            content: None,
+            msg: Some(err.to_string()),
+        },
+    };
+
+    return Ok(Json(response));
+}
+
 async fn get_workspace_s3_resource<'c>(
     authed: &ApiAuthed,
     user_db: &UserDB,
@@ -261,20 +648,37 @@ async fn get_workspace_s3_resource<'c>(
         LargeFileStorage::S3Storage(s3_lfs) => s3_lfs,
     };
 
-    let resource_path_json_value = serde_json::to_value(s3_lfs.s3_resource_path)
-        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
-    let interpolated_value = transform_json_value(
+    let stripped_resource_path = match s3_lfs.s3_resource_path.strip_prefix("$res:") {
+        Some(stripped) => stripped,
+        None => s3_lfs.s3_resource_path.as_str(),
+    };
+    return get_s3_resource(authed, user_db, db, token, w_id, stripped_resource_path).await;
+}
+
+async fn get_s3_resource<'c>(
+    authed: &ApiAuthed,
+    user_db: &UserDB,
+    db: &DB,
+    token: &str,
+    w_id: &str,
+    s3_resource_path: &str,
+) -> error::Result<Option<S3Resource>> {
+    let s3_resource_value_raw = get_resource_value_interpolated_internal(
         authed,
         user_db,
         db,
-        &w_id,
-        resource_path_json_value,
-        &Option::None,
+        w_id,
+        s3_resource_path,
+        None,
         token,
     )
     .await?;
 
-    let s3_resource = serde_json::from_value::<S3Resource>(interpolated_value)
+    if s3_resource_value_raw.is_none() {
+        return Err(error::Error::NotFound("Resource not found".to_string()));
+    }
+
+    let s3_resource = serde_json::from_value::<S3Resource>(s3_resource_value_raw.unwrap())
         .map_err(|err| error::Error::InternalErr(err.to_string()))?;
     return Ok(Some(s3_resource));
 }
@@ -282,18 +686,10 @@ async fn get_workspace_s3_resource<'c>(
 fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
     let s3_resource = s3_resource_ref.clone();
 
-    let endpoint_with_prefix = if s3_resource.endpoint.starts_with("http://")
-        || s3_resource.endpoint.starts_with("https://")
-    {
-        s3_resource.endpoint.clone()
-    } else if s3_resource.use_ssl {
-        format!("https://{}", s3_resource.endpoint)
-    } else {
-        format!("http://{}", s3_resource.endpoint)
-    };
-
+    let endpoint = render_endpoint(&s3_resource);
     let mut s3_config_builder = aws_sdk_s3::Config::builder()
-        .endpoint_url(endpoint_with_prefix)
+        .endpoint_url(endpoint)
+        .behavior_version(BehaviorVersion::latest())
         .region(Region::new(s3_resource.region));
     if s3_resource.access_key.is_some() {
         s3_config_builder = s3_config_builder.credentials_provider(Credentials::new(
@@ -310,3 +706,199 @@ fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
     let s3_config = s3_config_builder.build();
     return aws_sdk_s3::Client::from_conf(s3_config);
 }
+
+fn build_polars_s3_config(s3_resource_ref: &S3Resource) -> CloudOptions {
+    let s3_resource = s3_resource_ref.to_owned();
+    let mut s3_configs: Vec<(AmazonS3ConfigKey, String)> = vec![
+        (AmazonS3ConfigKey::Region, s3_resource.region),
+        (AmazonS3ConfigKey::Bucket, s3_resource.bucket),
+        (
+            AmazonS3ConfigKey::Endpoint,
+            render_endpoint(s3_resource_ref),
+        ),
+        (
+            AmazonS3ConfigKey::Client(ClientConfigKey::AllowHttp),
+            (!s3_resource.use_ssl).to_string(),
+        ),
+        (
+            AmazonS3ConfigKey::VirtualHostedStyleRequest,
+            (!s3_resource.path_style).to_string(),
+        ),
+    ];
+    if let Some(access_key) = s3_resource.access_key {
+        s3_configs.push((AmazonS3ConfigKey::AccessKeyId, access_key));
+    }
+    if let Some(secret_key) = s3_resource.secret_key {
+        s3_configs.push((AmazonS3ConfigKey::SecretAccessKey, secret_key));
+    }
+    return CloudOptions::default().with_aws(s3_configs);
+}
+
+fn render_endpoint(s3_resource: &S3Resource) -> String {
+    if s3_resource.endpoint.starts_with("http://") || s3_resource.endpoint.starts_with("https://") {
+        s3_resource.endpoint.clone()
+    } else if s3_resource.use_ssl {
+        format!("https://{}", s3_resource.endpoint)
+    } else {
+        format!("http://{}", s3_resource.endpoint)
+    }
+}
+
+async fn read_s3_text_object_head(
+    s3_client: &aws_sdk_s3::Client,
+    s3_bucket: &str,
+    file_key: &str,
+    from_char: i64,
+    length: i64,
+) -> error::Result<String> {
+    let s3_object = s3_client
+        .get_object()
+        .range(format!("bytes={}-{}", from_char, length).to_string())
+        .bucket(s3_bucket)
+        .key(file_key)
+        .send()
+        .await
+        .map_err(|err| {
+            tracing::warn!("Error fetching text file from S3: {:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
+
+    let payload = s3_object
+        .body
+        .collect()
+        .await
+        .map_err(|err| {
+            tracing::warn!(
+                "Error reading raw text file {}. Error was: {:?}",
+                file_key,
+                err
+            );
+            error::Error::InternalErr("File encoding is not supported".to_string())
+        })?
+        .into_bytes()
+        .to_vec();
+
+    let file_header_str = String::from_utf8(payload).map_err(|err| {
+        tracing::warn!(
+            "Encoding of file {} unsupported. Error was: {:?}",
+            file_key,
+            err
+        );
+        error::Error::InternalErr("File encoding is not supported".to_string())
+    })?;
+    return Ok(file_header_str);
+}
+
+async fn read_s3_parquet_object_head(
+    s3_resource_ref: &S3Resource,
+    file_key: &str,
+) -> error::Result<String> {
+    let s3_cloud_config = build_polars_s3_config(s3_resource_ref);
+
+    let args: ScanArgsParquet = ScanArgsParquet {
+        n_rows: Some(1),
+        cache: false,
+        parallel: polars::io::parquet::ParallelStrategy::Auto,
+        rechunk: false,
+        row_count: None,
+        low_memory: false,
+        use_statistics: false,
+        hive_partitioning: false,
+        cloud_options: Some(s3_cloud_config),
+    };
+
+    let file_key_clone = file_key.to_string();
+    let s3_bucket_clone = s3_resource_ref.bucket.to_string();
+    let polars_df_result = tokio::task::spawn_blocking(move || {
+        let s3_file_key = format!("s3://{}/{}", s3_bucket_clone, file_key_clone);
+        let lzdf_result = LazyFrame::scan_parquet(s3_file_key, args);
+        match lzdf_result {
+            Err(err) => {
+                tracing::warn!("Error fetching parquet file from S3: {:?}", err);
+                return Err(error::Error::InternalErr(err.to_string()));
+            }
+            Ok(lzdf) => {
+                let df = lzdf
+                    .select(&[col("*")])
+                    .limit(10) // for now read only first 10 lines
+                    .collect()
+                    .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+                return Ok(format!("{:?}", df).to_string());
+            }
+        }
+    })
+    .await
+    .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+
+    return polars_df_result;
+}
+
+async fn csv_file_preview_with_fallback(
+    s3_client: &aws_sdk_s3::Client,
+    s3_bucket: &str,
+    file_key: &str,
+    length: i64,
+    separator: Option<String>,
+    has_header: Option<bool>,
+) -> error::Result<String> {
+    match read_s3_csv_object_head(
+        &s3_client, &s3_bucket, &file_key, length, separator, has_header,
+    )
+    .await
+    {
+        Ok(csv_preview) => Ok(csv_preview),
+        Err(_) => {
+            // fallback to default text file preview is the CSV could not be parsed. It's a text file after all
+            let raw_text =
+                read_s3_text_object_head(&s3_client, &s3_bucket, &file_key, 0, length).await?;
+            return Ok(raw_text);
+        }
+    }
+}
+
+async fn read_s3_csv_object_head(
+    s3_client: &aws_sdk_s3::Client,
+    s3_bucket: &str,
+    file_key: &str,
+    length: i64,
+    separator: Option<String>,
+    has_header: Option<bool>,
+) -> error::Result<String> {
+    let separator_final = match separator {
+        Some(separator_char) if separator_char == "\\t" => Ok("\t".as_bytes()[0]),
+        Some(separator_char) if separator_char.len() != 1 => Err(error::Error::BadRequest(
+            "Separator must be a single character".to_string(),
+        )),
+        Some(separator_char) => Ok(separator_char.as_bytes()[0]),
+        None => Ok(",".as_bytes()[0]), // polars uses the comma as default, doing the same here
+    }?;
+
+    let s3_object = s3_client
+        .get_object()
+        .bucket(s3_bucket)
+        .range(format!("bytes=0-{}", length).to_string())
+        .key(file_key)
+        .send()
+        .await
+        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+
+    // TODO: polars does not seem to support lazy csv reader, unfortunately. We can implement it ourselves if needed
+    // Right now it's fine b/c we limit the download from AWS to 32MB. We should recomment users to use parquet
+    // for larger files
+    let file_content_bytes = s3_object
+        .body
+        .collect()
+        .await
+        .map_err(|err| error::Error::InternalErr(err.to_string()))?
+        .into_bytes();
+    let cursor = std::io::Cursor::new(file_content_bytes);
+
+    let csv_df = CsvReader::new(cursor)
+        .with_n_rows(Some(10)) // for now read only first 10 lines
+        .with_separator(separator_final)
+        .has_header(has_header.unwrap_or(true))
+        .finish()
+        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+
+    return Ok(format!("{:?}", csv_df).to_string());
+}