Apply automatic changes

* chore(main): release 1.236.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/change-versions-mac.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+root_dirpath="$(cd "${script_dirpath}/.." && pwd)"
+
+VERSION=$1
+echo "Updating versions to: $VERSION"
+
+sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
+sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/"  ${root_dirpath}/cli/main.ts
+sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
+sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
+sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml
+sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json
+sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json
+sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
+sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
+sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
+# sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
+sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
+sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
+
+sed -i '' -E "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" ${root_dirpath}/backend/Cargo.lock
+
+cd ${root_dirpath}/frontend && npm i --package-lock-only

.github/change-versions.sh

@@ -1,22 +1,25 @@
 #!/bin/bash
+set -euo pipefail
+script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+root_dirpath="$(cd "${script_dirpath}/.." && pwd)"
 
 VERSION=$1
 echo "Updating versions to: $VERSION"
 
-sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
-sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
-sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
-sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json
-sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
-sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
-sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill/pyproject.toml
-sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill_pg/pyproject.toml
-# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
-sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
-sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" lsp/Pipfile
+sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/cli/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
+sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
+sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml
+sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json
+sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json
+sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
+sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
+sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
+# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
+sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
+sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
 
-sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" backend/Cargo.lock
+sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" ${root_dirpath}/backend/Cargo.lock
 
-cd frontend && npm i --package-lock-only
+cd ${root_dirpath}/frontend && npm i --package-lock-only

.github/workflows/docker-image.yml

@@ -43,8 +43,9 @@ jobs:
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=false
           tags: |
-            type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
@@ -56,7 +57,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
             ${{ steps.meta-public.outputs.tags }}
           labels: |
             ${{ steps.meta-public.outputs.labels }}
@@ -79,8 +80,9 @@ jobs:
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
+          flavor: |
+            latest=false
           tags: |
-            type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
@@ -102,7 +104,7 @@ jobs:
             features=enterprise,enterprise_saml
             nsjail=true
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
             ${{ steps.meta-ee-public.outputs.tags }}
           labels: |
             ${{ steps.meta-ee-public.outputs.labels }}
@@ -198,25 +200,25 @@ jobs:
 
       - run: |
           # pulling docker image with desired arch so that actions-docker-extract doesn't do it
-          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
 
       - run: |
           # Checks the image is in docker prior to running actions-docker-extract. It fails if not
           # Also useful to visually check that the arch is the right opencontainers
-          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
 
       - uses: shrink/actions-docker-extract@v3
         id: extract
         with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
           path: "/usr/src/app/windmill"
 
       - uses: shrink/actions-docker-extract@v3
         id: extract-ee
         with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
           path: "/usr/src/app/windmill"
 
       - name: Rename binary with corresponding architecture
@@ -242,25 +244,25 @@ jobs:
 
       - run: |
           # pulling docker image with desired arch so that actions-docker-extract doesn't do it
-          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
 
       - run: |
           # Checks the image is in docker prior to running actions-docker-extract. It fails if not
           # Also useful to visually check that the arch is the right opencontainers
-          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
 
       - uses: shrink/actions-docker-extract@v3
         id: extract
         with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
           path: "/usr/src/app/windmill"
 
       - uses: shrink/actions-docker-extract@v3
         id: extract-ee
         with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
           path: "/usr/src/app/windmill"
 
       - name: Rename binary with corresponding architecture
@@ -305,7 +307,7 @@ jobs:
           git_hash=$(git rev-parse --short "$GITHUB_SHA")
           docker buildx imagetools create \
             --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${git_hash:0:7} \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
           echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"
 
       - uses: shrink/actions-docker-extract@v3
@@ -322,3 +324,63 @@ jobs:
           folder: ${{ steps.extract.outputs.destination }}
           bucket: windmill-frontend
           bucket-region: us-east-1
+
+  run_integration_test:
+    runs-on: ubicloud
+    needs: [build_ee]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Prepare test run
+        if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
+        run: cd integration_tests && ./build.sh
+      - name: Test run
+        if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
+        timeout-minutes: 15
+        env:
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+        run: cd integration_tests && ./run.sh
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Windmill Integration Tests Logs
+          path: |
+            integration_tests/logs
+
+  tag_latest:
+    runs-on: ubicloud
+    needs: [run_integration_test]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Tag main and latest
+        run: |
+          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
+
+  tag_latest_ee:
+    runs-on: ubicloud
+    needs: [run_integration_test]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Tag main and latest for ee
+        run: |
+          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main

CHANGELOG.md

@@ -1,6 +1,183 @@
 # Changelog
 
 
+## [1.236.2](https://github.com/windmill-labs/windmill/compare/v1.236.1...v1.236.2) (2024-01-08)
+
+
+### Bug Fixes
+
+* handle better non object result for app result loader ([269afe4](https://github.com/windmill-labs/windmill/commit/269afe41f7e02da8d3a364e649e5fef00847d24a))
+
+## [1.236.1](https://github.com/windmill-labs/windmill/compare/v1.236.0...v1.236.1) (2024-01-08)
+
+
+### Bug Fixes
+
+* fix app initialization ([6b075e9](https://github.com/windmill-labs/windmill/commit/6b075e92ef4e312e6a4ce582dc7c4d88456b2f5d))
+
+## [1.236.0](https://github.com/windmill-labs/windmill/compare/v1.235.1...v1.236.0) (2024-01-08)
+
+
+### Features
+
+* improve git sync and recompute dependents when common python script is redeployed ([#2967](https://github.com/windmill-labs/windmill/issues/2967)) ([9449381](https://github.com/windmill-labs/windmill/commit/94493812ac0030cc6bba468d8ee24a14661716a8))
+
+
+### Bug Fixes
+
+* reduce stack size needed for workflow transitions ([4384617](https://github.com/windmill-labs/windmill/commit/4384617f1ec4c27de88ef6c895d6ef060a79f6bd))
+
+## [1.235.1](https://github.com/windmill-labs/windmill/compare/v1.235.0...v1.235.1) (2024-01-07)
+
+
+### Bug Fixes
+
+* fix resultjobloader ([2e02300](https://github.com/windmill-labs/windmill/commit/2e023003da8bea9efa86c0fbc52bf9723c73907d))
+
+## [1.235.0](https://github.com/windmill-labs/windmill/compare/v1.234.1...v1.235.0) (2024-01-06)
+
+
+### Features
+
+* **frontend:** DB Explorer ([#2892](https://github.com/windmill-labs/windmill/issues/2892)) ([fffc533](https://github.com/windmill-labs/windmill/commit/fffc5338ce76fda9a68b19f03cefe199cc09a905))
+
+## [1.234.1](https://github.com/windmill-labs/windmill/compare/v1.234.0...v1.234.1) (2024-01-05)
+
+
+### Bug Fixes
+
+* fix double view runs button ([9efcb53](https://github.com/windmill-labs/windmill/commit/9efcb53bcd903bf756c4beb92a5c7634e8e016ad))
+* fix typo "Wourker" -> "Worker" ([4385edc](https://github.com/windmill-labs/windmill/commit/4385edc6ffdb1b2e4544347928b4cf08faf1eaa1))
+
+## [1.234.0](https://github.com/windmill-labs/windmill/compare/v1.233.0...v1.234.0) (2024-01-05)
+
+
+### Features
+
+* Detailed job memory footprint on job run page ([#2956](https://github.com/windmill-labs/windmill/issues/2956)) ([0710ce6](https://github.com/windmill-labs/windmill/commit/0710ce6ef25d8f2344ac1350e96aa906f16d1be1))
+* git sync can now push commits to individual branches ([#2959](https://github.com/windmill-labs/windmill/issues/2959)) ([fb5cd6a](https://github.com/windmill-labs/windmill/commit/fb5cd6a4298c88e26481b76258bf5b09a11cf4b0))
+
+
+### Bug Fixes
+
+* Deleting workspace delete all workspace related values in DB ([#2961](https://github.com/windmill-labs/windmill/issues/2961)) ([c05d119](https://github.com/windmill-labs/windmill/commit/c05d119ae5dfb6f1bae88fc9d53b48a707cd2f15))
+
+## [1.233.0](https://github.com/windmill-labs/windmill/compare/v1.232.0...v1.233.0) (2024-01-03)
+
+
+### Features
+
+* **frontend:** add the view runs buttons for operators ([#2932](https://github.com/windmill-labs/windmill/issues/2932)) ([821d2d7](https://github.com/windmill-labs/windmill/commit/821d2d7ea15e828dbb3683c85025dbc09c7e8dc7))
+* Job caching now works with S3 files ([#2934](https://github.com/windmill-labs/windmill/issues/2934)) ([b44618f](https://github.com/windmill-labs/windmill/commit/b44618f35b6fe4558a5cd9fddec1e82216a2cf5e))
+* PIP_LOCAL_DEPENDENCIES now accepts regex and is configurable from the UI ([#2940](https://github.com/windmill-labs/windmill/issues/2940)) ([e0140f1](https://github.com/windmill-labs/windmill/commit/e0140f175d371e239f315613a537820b9a6ad25f))
+
+
+### Bug Fixes
+
+* allow extraConfig for agGrid to set onGridReady ([74fc2d3](https://github.com/windmill-labs/windmill/commit/74fc2d3f4397d976fec326893f8196008cb8b332))
+* decision tree fix wording ([#2941](https://github.com/windmill-labs/windmill/issues/2941)) ([0299c65](https://github.com/windmill-labs/windmill/commit/0299c656c368876581dc228c76513f1171ffd682))
+* decision tree nits ([#2936](https://github.com/windmill-labs/windmill/issues/2936)) ([787017f](https://github.com/windmill-labs/windmill/commit/787017f53c8227e1246b09e412e92316977937ae))
+* **frontend:** handle case where large json has a file and a filename ([#2951](https://github.com/windmill-labs/windmill/issues/2951)) ([2494171](https://github.com/windmill-labs/windmill/commit/2494171b9cb3d1857235b6558d276451d5550560))
+* persist state for agGrid ([c085f5f](https://github.com/windmill-labs/windmill/commit/c085f5fa7ec598d0abf6aad98de0e5c7b1c92985))
+* Tag all docker images with latest and main tags ([#2953](https://github.com/windmill-labs/windmill/issues/2953)) ([9655718](https://github.com/windmill-labs/windmill/commit/965571853ca253b130bcc4473190b6c9c98f44e1))
+
+## [1.232.0](https://github.com/windmill-labs/windmill/compare/v1.231.0...v1.232.0) (2023-12-27)
+
+
+### Features
+
+* add summary to schedules ([606b03d](https://github.com/windmill-labs/windmill/commit/606b03d7613cf15ab9a60ef34a82657ff49ccdbd))
+* no overlap for flows togglable on schedules ([4935528](https://github.com/windmill-labs/windmill/commit/49355280f6442c342a46f90b2fe2cb089b3f4eff))
+
+
+### Bug Fixes
+
+* fix file input ([263f41c](https://github.com/windmill-labs/windmill/commit/263f41cda3dba24bcaa2bbabfc3f5bc54c70f9cb))
+* improve ata for bun ([b66fcbf](https://github.com/windmill-labs/windmill/commit/b66fcbf6261f69a57e6711fa8b9d1d499c1b47ed))
+* use bun install in every case ([948b0da](https://github.com/windmill-labs/windmill/commit/948b0da59e6bcd3b6312373a1bbe52d541e54d56))
+
+## [1.231.0](https://github.com/windmill-labs/windmill/compare/v1.230.0...v1.231.0) (2023-12-26)
+
+
+### Features
+
+* bun can read private npm package ([#2915](https://github.com/windmill-labs/windmill/issues/2915)) ([5fa5ab1](https://github.com/windmill-labs/windmill/commit/5fa5ab1514a2d20083d30809c30e50bfcb29e82f))
+* **cli:** support skip args for push --raw ([fc07d58](https://github.com/windmill-labs/windmill/commit/fc07d5886893c8d493c3ca507e0c9d4a066622fc))
+
+
+### Bug Fixes
+
+* improve conditional wrapper ([4b65594](https://github.com/windmill-labs/windmill/commit/4b65594cbd138452c471a339aaa548c27fc0a4ce))
+
+## [1.230.0](https://github.com/windmill-labs/windmill/compare/v1.229.0...v1.230.0) (2023-12-22)
+
+
+### Features
+
+* add button for auto-formatting ([642c037](https://github.com/windmill-labs/windmill/commit/642c037c1a87efd8d63558cc0ca5e98fb7b51ff7))
+* move S3 file within bucket ([#2913](https://github.com/windmill-labs/windmill/issues/2913)) ([e40787c](https://github.com/windmill-labs/windmill/commit/e40787c616ec288ffc83450fac6f6f7f127ac008))
+* quick search in flow support workspace scripts ([5698a0e](https://github.com/windmill-labs/windmill/commit/5698a0ebf2052fd51b687951595a543f0b6c0308))
+* Retry for scheduled scripts ([#2893](https://github.com/windmill-labs/windmill/issues/2893)) ([5f60d46](https://github.com/windmill-labs/windmill/commit/5f60d468365c65dc48e247107487f64adf77b123))
+* S3 delete file and upload new ([#2909](https://github.com/windmill-labs/windmill/issues/2909)) ([e27e887](https://github.com/windmill-labs/windmill/commit/e27e8874918af84d8afffb26abd5828b4c43ed54))
+
+
+### Bug Fixes
+
+* add relative path handling for bun ATA ([6b570a7](https://github.com/windmill-labs/windmill/commit/6b570a779fa7592c527cee84b0ca238872d86347))
+* edit schedule from trigger page ([9667eb7](https://github.com/windmill-labs/windmill/commit/9667eb74f99b74621e0f00e3f380bbd34dd9654f))
+* Failing jobs in dedicated worker mode are now marked as failing ([#2894](https://github.com/windmill-labs/windmill/issues/2894)) ([5f85b67](https://github.com/windmill-labs/windmill/commit/5f85b67dfcf063fd8a3c3f69f0e7605fc40e473d))
+* fix multiplayer ([b5f1f99](https://github.com/windmill-labs/windmill/commit/b5f1f99daa30627f224f262653e459799fc0d6fe))
+* **frontend:** Fix debug condition + decision tree + add missing date to job preview + prevent header node from being selected in the flow editor ([#2912](https://github.com/windmill-labs/windmill/issues/2912)) ([d7b777a](https://github.com/windmill-labs/windmill/commit/d7b777a8e9b4a2a4f0772f9b29851476f9405dae))
+* improve javascript editors with semantic validation ([b3527f5](https://github.com/windmill-labs/windmill/commit/b3527f5164f9e487840773df57846394f118ebbb))
+* pin deno windmill-client version to relase ([ea322d9](https://github.com/windmill-labs/windmill/commit/ea322d9c14cc92b54cfc39791e162f8f248e74ab))
+* Python wrapper catches BaseException instead of Exception ([#2902](https://github.com/windmill-labs/windmill/issues/2902)) ([4c3c988](https://github.com/windmill-labs/windmill/commit/4c3c988f7bd35f290b9d051369029e790b1e5c5b))
+* remove bad app type forcing ([#2906](https://github.com/windmill-labs/windmill/issues/2906)) ([187c34a](https://github.com/windmill-labs/windmill/commit/187c34a6835e3e748bc609fb4096f42bf3e09a73))
+* Update S3 TS code snippets and Python SDK for Polars 0.20.X ([#2911](https://github.com/windmill-labs/windmill/issues/2911)) ([3cbdd89](https://github.com/windmill-labs/windmill/commit/3cbdd890cd2aaef1d983da0270f8c230c9b9285e))
+
+## [1.229.0](https://github.com/windmill-labs/windmill/compare/v1.228.1...v1.229.0) (2023-12-21)
+
+
+### Features
+
+* **frontend:** add a quick button to open decision tree graph ([#2897](https://github.com/windmill-labs/windmill/issues/2897)) ([3f7c19f](https://github.com/windmill-labs/windmill/commit/3f7c19f267a9965f21cf8a08040a45e088d24918))
+
+
+### Bug Fixes
+
+* **frontend:** fix save draft when the app has no versions ([#2904](https://github.com/windmill-labs/windmill/issues/2904)) ([0170fcc](https://github.com/windmill-labs/windmill/commit/0170fcc338e0b59f8af1894d868f3819cad8aca3))
+* Re-add TS SDK changes for S3 ([#2899](https://github.com/windmill-labs/windmill/issues/2899)) ([5fafd71](https://github.com/windmill-labs/windmill/commit/5fafd71135c5d4d1764e9f96984bc5f5e5810465))
+
+## [1.228.1](https://github.com/windmill-labs/windmill/compare/v1.228.0...v1.228.1) (2023-12-20)
+
+
+### Bug Fixes
+
+* TS SDK not found upon install error ([#2895](https://github.com/windmill-labs/windmill/issues/2895)) ([46e9818](https://github.com/windmill-labs/windmill/commit/46e9818c001b040c462ffe0b6f8681a97113ff04))
+
+## [1.228.0](https://github.com/windmill-labs/windmill/compare/v1.227.1...v1.228.0) (2023-12-20)
+
+
+### Features
+
+* Add Zitadel IDP ([#2888](https://github.com/windmill-labs/windmill/issues/2888)) ([afbea19](https://github.com/windmill-labs/windmill/commit/afbea19421572f2986bf8ab1e5817f1902c0a22c))
+* bun automatic type acquisition in frontend directly ([#2884](https://github.com/windmill-labs/windmill/issues/2884)) ([d507ce4](https://github.com/windmill-labs/windmill/commit/d507ce449b04bf0be5ea29ec2431ad1eea405f13))
+* not on latest app version warning ([#2889](https://github.com/windmill-labs/windmill/issues/2889)) ([e8b2cc8](https://github.com/windmill-labs/windmill/commit/e8b2cc8835881a7b2c0a273cb8e505644e9be001))
+* S3 objects are now typed in Python and TS SDK ([#2878](https://github.com/windmill-labs/windmill/issues/2878)) ([2155a6c](https://github.com/windmill-labs/windmill/commit/2155a6c1334b79272742ba4626abd9572c9130d2))
+
+
+### Bug Fixes
+
+* Persistent script spinner ([#2879](https://github.com/windmill-labs/windmill/issues/2879)) ([5dd5809](https://github.com/windmill-labs/windmill/commit/5dd58094fde38f13476e79b539e670ee150306d7))
+* Python imports from git repository ([#2886](https://github.com/windmill-labs/windmill/issues/2886)) ([292b230](https://github.com/windmill-labs/windmill/commit/292b2309a7b7093c85614ac9764620c9dce0e316))
+* remove 0x00 from logs automatically from bashoutput ([59397e7](https://github.com/windmill-labs/windmill/commit/59397e7445f78cca2e602d8b45591a0bb123a734))
+
+## [1.227.1](https://github.com/windmill-labs/windmill/compare/v1.227.0...v1.227.1) (2023-12-17)
+
+
+### Bug Fixes
+
+* improve on click behavior of tree view ([fa7261f](https://github.com/windmill-labs/windmill/commit/fa7261f273b517acd85a8c3f465cd8095cdf9f3c))
+
 ## [1.227.0](https://github.com/windmill-labs/windmill/compare/v1.226.1...v1.227.0) (2023-12-17)
 
 

Dockerfile

@@ -209,7 +209,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:1.0.8 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.18 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

README.md

@@ -2,7 +2,7 @@
   <a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
+Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
 
 <p align=center>
 Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL.

backend/.sqlx/query-08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM account WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "08c943c4b9b76da43b8b754e5c66ab3b6397e7edc82666662a8ea466496290fb"
+}

backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json

@@ -31,7 +31,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }

backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json

@@ -46,7 +46,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }

backend/.sqlx/query-266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "git_sync",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "266741894b9a6ed650cf999b6e5b7c797424f8291517d3abf096fe45548a0273"
+}

backend/.sqlx/query-2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text",
+        "TimestamptzArray",
+        "Float4Array"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2ce7a562b1de63f767758642a4864ed982b74b53a143901480058f957b9641dc"
+}

backend/.sqlx/query-30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM dependency_map\n                             WHERE importer_path = $1 AND importer_kind = 'script'\n                             AND workspace_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "30e9250edd0c4c6025322f51b758174435d6df4f835cb0883ba40897cb99a410"
+}

backend/.sqlx/query-4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -110,6 +110,21 @@
       },
       {
         "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 24,
         "name": "jobs",
         "type_info": "JsonArray"
       }
@@ -143,8 +158,11 @@
       true,
       true,
       false,
+      true,
+      true,
+      false,
       null
     ]
   },
-  "hash": "7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097"
+  "hash": "4422b7183ede17a9cbde4afae41be4da5447020e39398deedbcca9121492834a"
 }

backend/.sqlx/query-4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM token WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4c5276bbf6aaad004a58aaff6b2abd7e78903a0354dd041fcc0cc7cca2b60cc1"
+}

backend/.sqlx/query-5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)\n                                 VALUES ($1, $2, 'script', $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5fbd5dfb8df9aecce023c5d482ee58efcbc23c8682a3f0ec94977eb56023ed72"
+}

backend/.sqlx/query-60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM input WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "60cd5ea09b2eba9df97ab011dd8575940a9dd5e244f3e877d3802fc88087234b"
+}

backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json

@@ -41,7 +41,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }

backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a"
+}

backend/.sqlx/query-699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "699427e7e3e6cd3ba4f61f77e171c5c27234cfa30460696ac619760667d7553c"
+}

backend/.sqlx/query-6bb9e6cab7034bf28ef2597445dc617b60c49b42568fdb975cf2d2dc8702167e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING *",
+  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry, summary, no_flow_overlap ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21 ) RETURNING *",
   "describe": {
     "columns": [
       {
@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -128,6 +143,9 @@
         "Varchar",
         "Int4",
         "Json",
+        "Bool",
+        "Jsonb",
+        "Varchar",
         "Bool"
       ]
     },
@@ -152,8 +170,11 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },
-  "hash": "47a01c0924df7bc2695f8238425aa7eadf845e4156da8cf645582737e046589d"
+  "hash": "6bb9e6cab7034bf28ef2597445dc617b60c49b42568fdb975cf2d2dc8702167e"
 }

backend/.sqlx/query-73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text",
+        "TimestamptzArray",
+        "Int4Array"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "73a3a742702c1c8237a3216523dffe487146b14512d4882179978efe280956aa"
+}

backend/.sqlx/query-74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "74949628ea78a5fecfd415e6406636a9442edd5b7939f61894288103be0c03ef"
+}

backend/.sqlx/query-76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM dependency_map WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "76d73f2fe5ac75e5033fb93d60f2ef5ab2b0b73a80dcb6a250c17c3b2f15c6b4"
+}

backend/.sqlx/query-813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM deployment_metadata WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "813d3c65d4c4bc10f1ec16ed4ee0954f4b938f45751ac03fd45479e665db1d40"
+}

backend/.sqlx/query-911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -137,6 +152,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM raw_app WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "96a77e839d6210629234879eeeaa23d9bddf5f751fe27d999659f6597f9759f1"
+}

backend/.sqlx/query-96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -138,6 +153,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -25,7 +25,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }

backend/.sqlx/query-aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -136,6 +151,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT importer_path FROM dependency_map\n         WHERE imported_path = $1 AND importer_kind = 'script'\n         AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "importer_path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "aa44311f9bae8253c6c483d122597ce7e470c6ba1905e7e877dca35d9d68968f"
+}

backend/.sqlx/query-aeedd3d92f7c50fe49d14053afebd82837093636da2950915ce4364106f5679c.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) FROM queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "aeedd3d92f7c50fe49d14053afebd82837093636da2950915ce4364106f5679c"
+}

backend/.sqlx/query-b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text",
+        "Float4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b56111c5b81de2449a1b687656a4636457da851a4db5e916cf4f6847c48c6bd3"
+}

backend/.sqlx/query-b7ed211ec7611c994a49565e46e5893788bcf49b2c9357fbc45a8353a7c84f48.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11 \n        WHERE path  = $12 AND workspace_id = $13 RETURNING *",
+  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11, retry = $12, summary = $13, no_flow_overlap = $14 WHERE path = $15 AND workspace_id = $16 RETURNING *",
   "describe": {
     "columns": [
       {
@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -122,6 +137,9 @@
         "Int4",
         "Json",
         "Bool",
+        "Jsonb",
+        "Varchar",
+        "Bool",
         "Text",
         "Text"
       ]
@@ -147,8 +165,11 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },
-  "hash": "2157813137b383411227b06814143df2d2187d1d43b84f67d22c0a2365b42ada"
+  "hash": "b7ed211ec7611c994a49565e46e5893788bcf49b2c9357fbc45a8353a7c84f48"
 }

backend/.sqlx/query-c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -137,6 +152,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM job_stats WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d77223fc94257711512b6ff21ce109b97324f2917113f04492f53b20860e9588"
+}

backend/.sqlx/query-dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -137,6 +152,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "deb2e4c21d4f9d0b9439780ca6ebee13450609b578c469cb437c5420ab520258"
+}

backend/.sqlx/query-eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -137,6 +152,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/.sqlx/query-f2173822c1b3d82afedf0f7dba31e218539f0b259eb5f2c98e4e8ada6665f5ee.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "f2173822c1b3d82afedf0f7dba31e218539f0b259eb5f2c98e4e8ada6665f5ee"
+}

backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json

@@ -40,7 +40,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }
@@ -63,7 +64,8 @@
                 "postgresql",
                 "noop",
                 "appdependencies",
-                "deploymentcallback"
+                "deploymentcallback",
+                "singlescriptflow"
               ]
             }
           }

backend/.sqlx/query-fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480.json

@@ -107,6 +107,21 @@
         "ordinal": 20,
         "name": "ws_error_handler_muted",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "retry",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 22,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 23,
+        "name": "no_flow_overlap",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -135,6 +150,9 @@
       true,
       true,
       true,
+      false,
+      true,
+      true,
       false
     ]
   },

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.227.0"
+version = "1.236.2"
 authors.workspace = true
 edition.workspace = true
 
@@ -11,6 +11,7 @@ members = [
     "./windmill-worker",
     "./windmill-common",
     "./windmill-audit",
+    "./windmill-git-sync",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
     "./parsers/windmill-parser-wasm",
@@ -21,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.227.0"
+version = "1.236.2"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -34,7 +35,7 @@ opt-level = 0
 incremental = true
 
 [features]
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml"]
 benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
@@ -46,6 +47,7 @@ anyhow.workspace = true
 tokio.workspace = true
 dotenv.workspace = true
 windmill-common = { workspace = true, features = ["tracing_init"] }
+windmill-git-sync.workspace = true
 windmill-api.workspace = true
 windmill-worker.workspace = true
 futures.workspace = true
@@ -66,7 +68,7 @@ uuid.workspace = true
 gethostname.workspace = true
 serde_json.workspace = true
 serde.workspace = true
-pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true} 
+pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false} 
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -83,6 +85,7 @@ windmill-queue = { path = "./windmill-queue" }
 windmill-worker = { path = "./windmill-worker" }
 windmill-common = { path = "./windmill-common" }
 windmill-audit = { path = "./windmill-audit" }
+windmill-git-sync = { path = "./windmill-git-sync" }
 windmill-parser = { path = "./parsers/windmill-parser" }
 windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
 windmill-parser-py = { path = "./parsers/windmill-parser-py" }
@@ -153,7 +156,7 @@ swc_ecma_visit = "0.93.7"
 base64 = "0.21.0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
-sqlx = { version = "^0", features = [
+sqlx = { version = "0.7.3", features = [
     "macros",
     "migrate",
     "uuid",
@@ -208,4 +211,7 @@ candle-transformers = "0.3.0"
 candle-nn = "0.3.0"
 tiberius = { version = "0.12.2", default-features = false, features = ["rustls", "tds73", "chrono"] }
 aws-sdk-s3 = "1.5.0"
-aws-config = "1.0.3"
+aws-config = "1.0.3"
+polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] }
+polars-io = { version = "0.35.2", features = ["csv"] }
+object_store = { version = "0.8.0", features = ["aws"] }

backend/migrations/20231219144259_schedule_retry.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE schedule DROP COLUMN IF EXISTS retry;

backend/migrations/20231219144259_schedule_retry.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS retry JSONB;
+ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'singlescriptflow';

backend/migrations/20231226062524_add_schedule_summary_no_overlap.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231226062524_add_schedule_summary_no_overlap.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN summary VARCHAR(512);
+ALTER TABLE schedule ADD COLUMN no_flow_overlap BOOLEAN NOT NULL DEFAULT FALSE;

backend/migrations/20231227014107_add_dependency_map.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE dependency_map;

backend/migrations/20231227014107_add_dependency_map.up.sql

@@ -0,0 +1,14 @@
+-- Add up migration script here
+
+CREATE TYPE IMPORTER_KIND AS ENUM ('script', 'flow', 'app');
+
+CREATE TABLE IF NOT EXISTS dependency_map (
+    workspace_id VARCHAR(50) NOT NULL,
+    importer_path VARCHAR(510) NOT NULL,
+    importer_kind IMPORTER_KIND NOT NULL,
+    imported_path VARCHAR(510) NOT NULL,
+    PRIMARY KEY (workspace_id, importer_path, importer_kind, imported_path)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS dependency_map_imported_path_idx ON dependency_map (workspace_id, imported_path);
+

backend/migrations/20240102133412_job_stats_table.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+DROP TABLE job_stats;
+DROP TYPE METRIC_KIND;

backend/migrations/20240102133412_job_stats_table.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+CREATE TYPE METRIC_KIND AS ENUM ('scalar_int', 'scalar_float', 'timeseries_int', 'timeseries_float');
+
+CREATE TABLE IF NOT EXISTS job_stats (
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+	job_id UUID NOT NULL,
+	metric_id VARCHAR(50) NOT NULL,
+	metric_name VARCHAR(255),
+	metric_kind METRIC_KIND NOT NULL,
+	scalar_int INTEGER,
+	scalar_float REAL,
+	timestamps TIMESTAMP WITH TIME ZONE[],
+	timeseries_int INTEGER[],
+	timeseries_float REAL[],
+    PRIMARY KEY (workspace_id, job_id, metric_id)
+);

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -63,7 +63,7 @@ lazy_static! {
 
 fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<String> {
     if level > 0 {
-        let mut imports = vec!["requests".to_string()];
+        let mut imports = vec![];
         let splitted_path = path.split("/");
         let base = splitted_path
             .clone()
@@ -78,10 +78,7 @@ fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<Strin
     } else if let Some(module) = module {
         let imprt = module.split('.').next().unwrap_or("").replace("_", "-");
         if imprt == "u" || imprt == "f" {
-            vec![
-                "requests".to_string(),
-                format!("relative:{}", module.replace(".", "/")),
-            ]
+            vec![format!("relative:{}", module.replace(".", "/"))]
         } else {
             vec![imprt]
         }
@@ -90,6 +87,57 @@ fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<Strin
     }
 }
 
+pub fn parse_relative_imports(code: &str, path: &str) -> error::Result<Vec<String>> {
+    let nimports = parse_code_for_imports(code, path)?;
+    return Ok(nimports
+        .into_iter()
+        .filter_map(|x| {
+            if x.starts_with("relative:") {
+                Some(x.replace("relative:", ""))
+            } else {
+                None
+            }
+        })
+        .collect());
+}
+
+fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>> {
+    let code = code.split(DEF_MAIN).next().unwrap_or("");
+    let ast = Suite::parse(code, "main.py").map_err(|e| {
+        error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+    })?;
+    let nimports: Vec<String> = ast
+        .into_iter()
+        .filter_map(|x| match x {
+            Stmt::Import(StmtImport { names, .. }) => Some(
+                names
+                    .into_iter()
+                    .map(|x| {
+                        let name = x.name.to_string();
+                        process_import(Some(name), path, 0)
+                    })
+                    .flatten()
+                    .collect::<Vec<String>>(),
+            ),
+            Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
+                Some(process_import(
+                    module.map(|x| x.to_string()),
+                    path,
+                    i.to_usize(),
+                ))
+            }
+            Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
+                Some(process_import(module.map(|x| x.to_string()), path, 0))
+            }
+            _ => None,
+        })
+        .flatten()
+        .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+        .unique()
+        .collect();
+    return Ok(nimports);
+}
+
 #[async_recursion]
 pub async fn parse_python_imports(
     code: &str,
@@ -127,41 +175,7 @@ pub async fn parse_python_imports(
             imports.extend(lines);
         }
 
-        let code = code.split(DEF_MAIN).next().unwrap_or("");
-        let ast = Suite::parse(code, "main.py").map_err(|e| {
-            error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
-        })?;
-        let nimports: Vec<String> = ast
-            .into_iter()
-            .filter_map(|x| match x {
-                Stmt::Import(StmtImport { names, .. }) => Some(
-                    names
-                        .into_iter()
-                        .map(|x| {
-                            let name = x.name.to_string();
-                            process_import(Some(name), path, 0)
-                        })
-                        .flatten()
-                        .collect::<Vec<String>>(),
-                ),
-                Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. })
-                    if i.to_u32() > 0 =>
-                {
-                    Some(process_import(
-                        module.map(|x| x.to_string()),
-                        path,
-                        i.to_usize(),
-                    ))
-                }
-                Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => {
-                    Some(process_import(module.map(|x| x.to_string()), path, 0))
-                }
-                _ => None,
-            })
-            .flatten()
-            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
-            .unique()
-            .collect();
+        let nimports = parse_code_for_imports(code, path)?;
         for n in nimports.iter() {
             let nested = if n.starts_with("relative:") {
                 let rpath = n.replace("relative:", "");

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -20,7 +20,7 @@ def main():
 ";
         let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
         // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
+        assert_eq!(r, vec!["matplotlib", "wmill", "zanzibar"]);
         Ok(())
     }
 
@@ -73,7 +73,6 @@ def main():
                 "numpy",
                 "pandas",
                 "pandas2",
-                "requests"
             ]
         );
 

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -47,10 +47,9 @@ impl Visit for ImportsFinder {
 
 pub fn parse_expr_for_imports(code: &str) -> anyhow::Result<Vec<String>> {
     let cm: Lrc<SourceMap> = Default::default();
-    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
+    let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()), code.into());
     let lexer = Lexer::new(
-        // We want to parse ecmascript
-        Syntax::Es(EsConfig { jsx: false, ..Default::default() }),
+        Syntax::Typescript(TsConfig::default()),
         // EsVersion defaults to es5
         Default::default(),
         StringInput::from(&*fm),
@@ -64,9 +63,9 @@ pub fn parse_expr_for_imports(code: &str) -> anyhow::Result<Vec<String>> {
         err_s += &e.into_kind().msg().to_string();
     }
 
-    let expr = parser
-        .parse_module()
-        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?;
+    let expr = parser.parse_module().map_err(|e| {
+        anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}")
+    })?;
 
     let mut visitor = ImportsFinder { imports: HashSet::new() };
     swc_ecma_visit::visit_module(&mut visitor, &expr);
@@ -119,9 +118,9 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
         err_s += &e.into_kind().msg().to_string();
     }
 
-    let expr = parser
-        .parse_module()
-        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?;
+    let expr = parser.parse_module().map_err(|e| {
+        anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}")
+    })?;
 
     let mut visitor = OutputFinder { idents: HashSet::new() };
     swc_ecma_visit::visit_module(&mut visitor, &expr);
@@ -150,7 +149,9 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainA
 
     let ast = parser
         .parse_module()
-        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?
+        .map_err(|e| {
+            anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}")
+        })?
         .body;
 
     let params = ast.into_iter().find_map(|x| match x {

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.222.0",
+  "version": "1.226.9",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts

@@ -14,6 +14,11 @@ export function parse_outputs(code: string): string;
 * @param {string} code
 * @returns {string}
 */
+export function parse_ts_imports(code: string): string;
+/**
+* @param {string} code
+* @returns {string}
+*/
 export function parse_bash(code: string): string;
 /**
 * @param {string} code
@@ -67,6 +72,7 @@ export interface InitOutput {
   readonly memory: WebAssembly.Memory;
   readonly parse_deno: (a: number, b: number, c: number) => void;
   readonly parse_outputs: (a: number, b: number, c: number) => void;
+  readonly parse_ts_imports: (a: number, b: number, c: number) => void;
   readonly parse_bash: (a: number, b: number, c: number) => void;
   readonly parse_powershell: (a: number, b: number, c: number) => void;
   readonly parse_go: (a: number, b: number, c: number) => void;

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -97,6 +97,15 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
 function addHeapObject(obj) {
     if (heap_next === heap.length) heap.push(heap.length + 1);
     const idx = heap_next;
@@ -115,15 +124,6 @@ function getFloat64Memory0() {
     return cachedFloat64Memory0;
 }
 
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
-}
-
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -243,6 +243,29 @@ export function parse_outputs(code) {
     }
 }
 
+/**
+* @param {string} code
+* @returns {string}
+*/
+export function parse_ts_imports(code) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.parse_ts_imports(retptr, ptr0, len0);
+        var r0 = getInt32Memory0()[retptr / 4 + 0];
+        var r1 = getInt32Memory0()[retptr / 4 + 1];
+        deferred2_0 = r0;
+        deferred2_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
+
 /**
 * @param {string} code
 * @returns {string}
@@ -518,10 +541,6 @@ function __wbg_get_imports() {
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
-    imports.wbg.__wbg_eval_1081105c41705556 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -530,6 +549,10 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -566,8 +589,8 @@ function __wbg_get_imports() {
         const ret = getObject(arg0) in getObject(arg1);
         return ret;
     };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
+    imports.wbg.__wbg_eval_596393dc5ae50a1b = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
         return addHeapObject(ret);
     };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts

@@ -3,6 +3,7 @@
 export const memory: WebAssembly.Memory;
 export function parse_deno(a: number, b: number, c: number): void;
 export function parse_outputs(a: number, b: number, c: number): void;
+export function parse_ts_imports(a: number, b: number, c: number): void;
 export function parse_bash(a: number, b: number, c: number): void;
 export function parse_powershell(a: number, b: number, c: number): void;
 export function parse_go(a: number, b: number, c: number): void;

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -31,7 +31,7 @@ pub fn parse_outputs(code: &str) -> String {
 pub fn parse_ts_imports(code: &str) -> String {
     let parsed = parse_expr_for_imports(code);
     let r = if let Ok(parsed) = parsed {
-        json!({ "imports6": parsed })
+        json!({ "imports": parsed })
     } else {
         json!({"error": parsed.err().unwrap().to_string()})
     };

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -298,3 +298,15 @@ fn test_parse_imports() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[wasm_bindgen_test]
+fn test_parse_imports_dts() -> anyhow::Result<()> {
+    let code = "
+export type foo = number
+";
+    let mut l = parse_expr_for_imports(code)?;
+    l.sort();
+    assert_eq!(l, vec![] as Vec<String>);
+
+    Ok(())
+}

backend/src/main.rs

@@ -431,7 +431,7 @@ Windmill Community Edition {GIT_VERSION}
                                                 },
                                                 REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
                                                     load_require_preexisting_user(&db).await;
-                                                }
+                                                },
                                                 EXPOSE_METRICS_SETTING | EXPOSE_DEBUG_METRICS_SETTING => {
                                                     if n.payload() != EXPOSE_DEBUG_METRICS_SETTING || worker_mode {
                                                         tracing::info!("Metrics setting changed, restarting");

backend/src/monitor.rs

@@ -35,8 +35,8 @@ use windmill_common::{
     BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
-    NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, SendResult, JOB_DEFAULT_TIMEOUT,
+    KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
 #[cfg(feature = "enterprise")]
@@ -167,6 +167,7 @@ pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
     };
     Ok(())
 }
+
 pub async fn load_keep_job_dir(db: &DB) {
     let value = sqlx::query_scalar!(
         "SELECT value FROM global_settings WHERE name = $1",
@@ -250,25 +251,52 @@ pub async fn delete_expired_items(db: &DB) -> () {
 
     let job_retention_secs = *JOB_RETENTION_SECS.read().await;
     if job_retention_secs > 0 {
-        let deleted_jobs = sqlx::query_scalar!(
-                "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval  AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
-                job_retention_secs
-            )
-            .fetch_all(db)
-            .await;
+        match db.begin().await {
+            Ok(mut tx) => {
+                let r = sqlx::query!(
+                    "DELETE FROM job_stats WHERE job_id IN (SELECT id FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now())",
+                    job_retention_secs
+                )
+                .fetch_all(&mut *tx)
+                .await;
+                match r {
+                    Ok(_) => {
+                        let deleted_jobs = sqlx::query_scalar!(
+                            "DELETE FROM completed_job WHERE created_at <= now() - ($1::bigint::text || ' s')::interval  AND started_at + ((duration_ms/1000 + $1::bigint) || ' s')::interval <= now() RETURNING id",
+                            job_retention_secs
+                        )
+                        .fetch_all(&mut *tx)
+                        .await;
 
-        match deleted_jobs {
-            Ok(deleted_jobs) => {
-                if deleted_jobs.len() > 0 {
-                    tracing::info!(
-                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
-                        deleted_jobs.len(),
-                        job_retention_secs,
-                        deleted_jobs,
-                    )
+                        match deleted_jobs {
+                            Ok(deleted_jobs) => {
+                                if deleted_jobs.len() > 0 {
+                                    tracing::info!(
+                                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+                                        deleted_jobs.len(),
+                                        job_retention_secs,
+                                        deleted_jobs,
+                                    )
+                                }
+                            }
+                            Err(e) => {
+                                tracing::error!("Error deleting expired jobs: {:?}", e)
+                            }
+                        }
+                    }
+                    Err(err) => {
+                        tracing::error!("Error deleting expired job stats: {:?}", err)
+                    }
+                }
+
+                match tx.commit().await {
+                    Ok(_) => (),
+                    Err(err) => tracing::error!("Error deleting expired jobs: {:?}", err),
                 }
             }
-            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+            Err(err) => {
+                tracing::error!("Error deleting expired jobs: {:?}", err)
+            }
         }
     }
 }
@@ -729,6 +757,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
 
         // since the job is unrecoverable, the same worker queue should never be sent anything
         let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
+        let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::<SendResult>(1);
 
         let token = create_token_for_owner(
             &db,
@@ -767,6 +796,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             "",
             rsmq.clone(),
             worker_name,
+            send_result_never_used,
         )
         .await;
     }

backend/tests/worker.rs

@@ -1805,7 +1805,7 @@ def main():
 }
 
 #[sqlx::test(fixtures("base"))]
-async fn test_empty_loop(db: Pool<Postgres>) {
+async fn test_empty_loop_1(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
     let port = server.addr.port();
@@ -2870,6 +2870,10 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
         timezone: "UTC".to_string(),
         schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
         ws_error_handler_muted: None,
+        retry: None,
+        no_flow_overlap: None,
+        summary: None,
+
     };
 
     let _ = client.create_schedule("test-workspace", &schedule).await;
@@ -2930,6 +2934,9 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
                 timezone: "UTC".to_string(),
                 schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
                 ws_error_handler_muted: None,
+                retry: None,
+                summary: None,
+                no_flow_overlap: None
             },
         )
         .await
@@ -3006,6 +3013,9 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
         timezone: "UTC".to_string(),
         schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
         ws_error_handler_muted: None,
+        retry: None,
+        no_flow_overlap: None,
+        summary: None,
     };
 
     let _ = client.create_schedule("test-workspace", &schedule).await;
@@ -3067,6 +3077,9 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
                 timezone: "UTC".to_string(),
                 schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
                 ws_error_handler_muted: None,
+                retry: None,
+                summary: None,
+                no_flow_overlap: None
             },
         )
         .await

backend/windmill-api/Cargo.toml

@@ -9,7 +9,7 @@ name = "windmill_api"
 path = "src/lib.rs"
 
 [features]
-enterprise = ["windmill-queue/enterprise", "async-stripe", "windmill-audit/enterprise"]
+enterprise = ["windmill-queue/enterprise",  "async-stripe", "windmill-audit/enterprise", "windmill-git-sync/enterprise"]
 enterprise_saml = ["samael"]
 benchmark = []
 
@@ -27,6 +27,7 @@ windmill-common = { workspace = true, features = [
 windmill-audit.workspace = true
 windmill-parser.workspace = true
 windmill-parser-py-imports.workspace = true
+windmill-git-sync.workspace = true
 tokio.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
@@ -84,7 +85,6 @@ candle-core.workspace = true
 candle-transformers.workspace = true
 candle-nn.workspace = true
 aws-sdk-s3.workspace = true
-aws-config.workspace = true
-polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] }
-polars-io = { version = "0.35.2", features = ["csv"] }
-object_store = { version = "0.8.0", features = ["aws"] }
+polars.workspace = true
+polars-io.workspace = true
+object_store.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.227.0
+  version: 1.236.2
   title: Windmill API
 
   contact:
@@ -2231,8 +2231,7 @@ paths:
 
   /w/{workspace}/resources/get_value_interpolated/{path}:
     get:
-      summary:
-        get resource interpolated (variables and resources are fully unrolled)
+      summary: get resource interpolated (variables and resources are fully unrolled)
       operationId: getResourceValueInterpolated
       tags:
         - resource
@@ -2920,8 +2919,7 @@ paths:
           schema:
             type: string
         - name: first_parent_hash
-          description:
-            mask to filter scripts whom first direct parent has exact hash
+          description: mask to filter scripts whom first direct parent has exact hash
           in: query
           schema:
             type: string
@@ -3116,8 +3114,7 @@ paths:
 
   /workers/custom_tags:
     get:
-      summary:
-        get all instance custom tags (tags are used to dispatch jobs to
+      summary: get all instance custom tags (tags are used to dispatch jobs to
         different worker groups)
       operationId: getCustomTags
       tags:
@@ -3168,8 +3165,7 @@ paths:
 
   /w/{workspace}/scripts/delete/h/{hash}:
     post:
-      summary:
-        delete script by hash (erase content but keep hash, require admin)
+      summary: delete script by hash (erase content but keep hash, require admin)
       operationId: deleteScriptByHash
       tags:
         - script
@@ -3408,8 +3404,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -3417,8 +3412,7 @@ paths:
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/NewJobId"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4451,8 +4445,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4461,8 +4454,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
           in: query
           schema:
             type: boolean
@@ -4514,8 +4506,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4524,8 +4515,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
           in: query
           schema:
             type: boolean
@@ -4563,8 +4553,7 @@ paths:
             type: string
             format: date-time
         - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
           in: query
           schema:
             type: integer
@@ -4573,8 +4562,7 @@ paths:
         - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4605,8 +4593,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4639,8 +4626,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
           in: query
           schema:
             type: boolean
@@ -4857,8 +4843,7 @@ paths:
         - job
       responses:
         "200":
-          description:
-            the timestamp of the db that can be used to compute the drift
+          description: the timestamp of the db that can be used to compute the drift
           content:
             application/json:
               schema:
@@ -5122,8 +5107,7 @@ paths:
 
   /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
     get:
-      summary:
-        get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
       operationId: getResumeUrls
       tags:
         - job
@@ -5674,8 +5658,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the groups the user is member of (default false)
+          description: only list the groups the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -5863,8 +5846,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - name: only_member_of
           in: query
-          description:
-            only list the folders the user is member of (default false)
+          description: only list the folders the user is member of (default false)
           schema:
             type: boolean
       responses:
@@ -6670,7 +6652,7 @@ paths:
                       - use_ssl
                       - cache_regions
                       - client_kwargs
-                  polars_cloud_options:
+                  storage_options:
                     type: object
                     properties:
                       aws_endpoint_url:
@@ -6682,28 +6664,27 @@ paths:
                       aws_region:
                         type: string
                       aws_allow_http:
-                        type: boolean
+                        type: string
                     required:
                       - aws_endpoint_url
                       - aws_region
                       - aws_allow_http
                 required:
                   - s3fs_args
-                  - polars_cloud_options
-  /w/{workspace}/job_helpers/v2/boto3_connection_settings:
+                  - storage_options
+  /w/{workspace}/job_helpers/v2/s3_resource_info:
     post:
-      summary:
-        Converts an S3 resource to the set of arguments necessary to connect to
-        an S3 bucket using boto3 in Python
-      operationId: boto3ConnectionSettingsV2
+      summary: Returns the s3 resource associated to the provided path, or the
+        workspace default S3 resource
+      operationId: s3ResourceInfo
       tags:
         - helpers
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
         description:
-          S3 resource path to use to generate the connection settings. If empty,
-          the S3 resource defined in the workspace settings will be used
+          S3 resource path to use. If empty, the S3 resource defined in the
+          workspace settings will be used
         required: true
         content:
           application/json:
@@ -6718,22 +6699,7 @@ paths:
           content:
             application/json:
               schema:
-                type: object
-                properties:
-                  endpoint_url:
-                    type: string
-                  region_name:
-                    type: string
-                  use_ssl:
-                    type: boolean
-                  aws_access_key_id:
-                    type: string
-                  aws_secret_access_key:
-                    type: string
-                required:
-                  - endpoint_url
-                  - region_name
-                  - use_ssl
+                $ref: "#/components/schemas/S3Resource"
 
   /w/{workspace}/job_helpers/test_connection:
     get:
@@ -6850,6 +6816,157 @@ paths:
               schema:
                 $ref: "#/components/schemas/WindmillFilePreview"
 
+  /w/{workspace}/job_helpers/delete_s3_file:
+    delete:
+      summary: Permanently delete file from S3
+      operationId: deleteS3File
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: file_key
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Confirmation
+          content:
+            application/json:
+              schema: {}
+
+  /w/{workspace}/job_helpers/move_s3_file:
+    get:
+      summary: Move a S3 file from one path to the other within the same bucket
+      operationId: moveS3File
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: src_file_key
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: dest_file_key
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Confirmation
+          content:
+            application/json:
+              schema: {}
+
+  /w/{workspace}/job_helpers/multipart_upload_s3_file:
+    post:
+      summary: Upload file to S3 bucket using multipart upload
+      operationId: multipartFileUpload
+      tags:
+        - helpers
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: Query args for a multipart file upload to S3
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                file_key:
+                  type: string
+                part_content:
+                  type: array
+                  items:
+                    type: integer
+                upload_id:
+                  type: string
+                parts:
+                  type: array
+                  items:
+                    $ref: "#/components/schemas/UploadFilePart"
+                is_final:
+                  type: boolean
+                cancel_upload:
+                  type: boolean
+              required:
+                - file_key
+                - part_content_base64
+                - parts
+                - is_final
+                - cancel_upload
+      responses:
+        "200":
+          description: Chunk upload status
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  upload_id:
+                    type: string
+                  parts:
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/UploadFilePart"
+                  is_done:
+                    type: boolean
+                required:
+                  - upload_id
+                  - parts
+                  - is_done
+
+  /w/{workspace}/job_metrics/get/{id}:
+    post:
+      summary: get job metrics
+      operationId: getJobMetrics
+      tags:
+        - metrics
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      requestBody:
+        description: parameters for statistics retrieval
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                timeseries_max_datapoints:
+                  type: integer
+                from_timestamp:
+                  type: string
+                  format: date-time
+                to_timestamp:
+                  type: string
+                  format: date-time
+
+      responses:
+        "200":
+          description: job details
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  metrics_metadata:
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/MetricMetadata"
+                  scalar_metrics:
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/ScalarMetric"
+                  timeseries_metrics:
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/TimeseriesMetric"
+
 components:
   securitySchemes:
     bearerAuth:
@@ -6948,8 +7065,7 @@ components:
         type: integer
     PerPage:
       name: per_page
-      description:
-        number of items to return for a given page (default 30, max 100)
+      description: number of items to return for a given page (default 30, max 100)
       in: query
       schema:
         type: integer
@@ -7077,8 +7193,7 @@ components:
         type: boolean
     ScheduledForBeforeNow:
       name: scheduled_for_before_now
-      description:
-        filter on jobs scheduled_for before now (hence waitinf for a worker)
+      description: filter on jobs scheduled_for before now (hence waitinf for a worker)
       in: query
       schema:
         type: boolean
@@ -7096,8 +7211,7 @@ components:
         type: boolean
     ArgsFilter:
       name: args
-      description:
-        filter on jobs containing those args as a json subset (@> in postgres)
+      description: filter on jobs containing those args as a json subset (@> in postgres)
       in: query
       schema:
         type: string
@@ -7109,8 +7223,7 @@ components:
         type: string
     ResultFilter:
       name: result
-      description:
-        filter on jobs containing those result as a json subset (@> in postgres)
+      description: filter on jobs containing those result as a json subset (@> in postgres)
       in: query
       schema:
         type: string
@@ -7517,6 +7630,7 @@ components:
               "script_hub",
               "identity",
               "deploymentcallback",
+              "singlescriptflow",
             ]
         schedule_path:
           type: string
@@ -7626,6 +7740,7 @@ components:
               "script_hub",
               "identity",
               "deploymentcallback",
+              "singlescriptflow",
             ]
         schedule_path:
           type: string
@@ -8332,6 +8447,12 @@ components:
           $ref: "#/components/schemas/ScriptArgs"
         ws_error_handler_muted:
           type: boolean
+        retry:
+          $ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
+        summary:
+          type: string
+        no_flow_overlap:
+          type: boolean
       required:
         - path
         - edited_by
@@ -8399,6 +8520,12 @@ components:
           $ref: "#/components/schemas/ScriptArgs"
         ws_error_handler_muted:
           type: boolean
+        retry:
+          $ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
+        no_flow_overlap:
+          type: boolean
+        summary:
+          type: string
       required:
         - path
         - schedule
@@ -8433,6 +8560,12 @@ components:
           $ref: "#/components/schemas/ScriptArgs"
         ws_error_handler_muted:
           type: boolean
+        retry:
+          $ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
+        no_flow_overlap:
+          type: boolean
+        summary:
+          type: string
       required:
         - schedule
         - timezone
@@ -8979,6 +9112,65 @@ components:
           type: string
         git_repo_resource_path:
           type: string
+        use_individual_branch:
+          type: boolean
       required:
         - script_path
         - git_repo_resource_path
+
+    UploadFilePart:
+      type: object
+      properties:
+        part_number:
+          type: integer
+        tag:
+          type: string
+      required:
+        - part_number
+        - tag
+
+    MetricMetadata:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+      required:
+        - id
+
+    ScalarMetric:
+      type: object
+      properties:
+        metric_id:
+          type: string
+        value:
+          type: number
+      required:
+        - id
+        - value
+
+    TimeseriesMetric:
+      type: object
+      properties:
+        metric_id:
+          type: string
+        values:
+          type: array
+          items:
+            $ref: "#/components/schemas/MetricDataPoint"
+      required:
+        - id
+        - values
+
+    MetricDataPoint:
+      type: object
+      properties:
+        timestamp:
+          type: string
+          format: date-time
+        value:
+          type: number
+      required:
+        - timestamp
+        - value

backend/windmill-api/src/apps.rs

@@ -9,7 +9,6 @@ use std::collections::HashMap;
  */
 use crate::{
     db::{ApiAuthed, DB},
-    deployment_metadata_helpers,
     users::{require_owner_of_path, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
@@ -588,12 +587,16 @@ async fn create_app(
     )
     .await?;
 
-    let mut tx = PushIsolationLevel::Transaction(tx);
+    let tx = PushIsolationLevel::Transaction(tx);
     let (dependency_job_uuid, new_tx) = push(
         &db,
         tx,
         &w_id,
-        JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
+        JobPayload::AppDependencies {
+            path: app.path.clone(),
+            version: v_id,
+            deployment_message: app.deployment_message,
+        },
         PushArgs::empty(),
         &authed.username,
         &authed.email,
@@ -615,25 +618,7 @@ async fn create_app(
     .await?;
     tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
 
-    tx = PushIsolationLevel::Transaction(new_tx);
-    tx = deployment_metadata_helpers::handle_deployment_metadata(
-        tx,
-        &authed,
-        &db,
-        &w_id,
-        deployment_metadata_helpers::DeployedObject::App { path: app.path.clone(), version: v_id },
-        app.deployment_message,
-    )
-    .await?;
-
-    match tx {
-        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
-        _ => {
-            return Err(Error::InternalErr(
-                "Expected a transaction here".to_string(),
-            ));
-        }
-    }
+    new_tx.commit().await?;
 
     webhook.send_message(
         w_id.clone(),
@@ -849,14 +834,18 @@ async fn update_app(
     )
     .await?;
 
-    let mut tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
+    let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
         PushIsolationLevel::Transaction(tx);
     if let Some(v_id) = v_id {
         let (dependency_job_uuid, new_tx) = push(
             &db,
             tx,
             &w_id,
-            JobPayload::AppDependencies { path: npath.clone(), version: v_id },
+            JobPayload::AppDependencies {
+                path: npath.clone(),
+                version: v_id,
+                deployment_message: ns.deployment_message,
+            },
             PushArgs::empty(),
             &authed.username,
             &authed.email,
@@ -877,26 +866,7 @@ async fn update_app(
         )
         .await?;
         tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
-
-        tx = PushIsolationLevel::Transaction(new_tx);
-        tx = deployment_metadata_helpers::handle_deployment_metadata(
-            tx,
-            &authed,
-            &db,
-            &w_id,
-            deployment_metadata_helpers::DeployedObject::App { path: npath.clone(), version: v_id },
-            ns.deployment_message,
-        )
-        .await?;
-    }
-
-    match tx {
-        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
-        _ => {
-            return Err(Error::InternalErr(
-                "Expected a transaction here".to_string(),
-            ));
-        }
+        new_tx.commit().await?;
     }
 
     webhook.send_message(

backend/windmill-api/src/audit.rs

@@ -25,10 +25,10 @@ pub fn workspaced_service() -> Router {
 async fn get_audit(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
-    Path(id): Path<i32>,
+    Path((w_id, id)): Path<(String, i32)>,
 ) -> JsonResult<AuditLog> {
     let tx = user_db.begin(&authed).await?;
-    let audit = windmill_audit::get_audit(tx, id).await?;
+    let audit = windmill_audit::get_audit(tx, id, &w_id).await?;
     Ok(Json(audit))
 }
 async fn list_audit(

backend/windmill-api/src/flows.rs

@@ -7,7 +7,6 @@
  */
 
 use crate::db::ApiAuthed;
-use crate::deployment_metadata_helpers;
 use crate::{
     db::DB,
     schedule::clear_schedule,
@@ -355,7 +354,7 @@ async fn create_flow(
     )
     .await?;
 
-    let mut tx = PushIsolationLevel::Transaction(tx);
+    let tx = PushIsolationLevel::Transaction(tx);
     let (dependency_job_uuid, mut new_tx) = push(
         &db,
         tx,
@@ -363,6 +362,7 @@ async fn create_flow(
         JobPayload::FlowDependencies {
             path: nf.path.clone(),
             dedicated_worker: nf.dedicated_worker,
+            deployment_message: nf.deployment_message,
         },
         PushArgs::empty(),
         &authed.username,
@@ -393,26 +393,7 @@ async fn create_flow(
     .execute(&mut new_tx)
     .await?;
 
-    tx = PushIsolationLevel::Transaction(new_tx);
-    tx = deployment_metadata_helpers::handle_deployment_metadata(
-        tx,
-        &authed,
-        &db,
-        &w_id,
-        deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
-        nf.deployment_message,
-    )
-    .await?;
-
-    match tx {
-        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
-        _ => {
-            return Err(Error::InternalErr(
-                "Expected a transaction here".to_string(),
-            ));
-        }
-    }
-
+    new_tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
@@ -585,7 +566,7 @@ async fn update_flow(
         },
     );
 
-    let mut tx = PushIsolationLevel::Transaction(tx);
+    let tx = PushIsolationLevel::Transaction(tx);
 
     let (dependency_job_uuid, mut new_tx) = push(
         &db,
@@ -594,6 +575,7 @@ async fn update_flow(
         JobPayload::FlowDependencies {
             path: nf.path.clone(),
             dedicated_worker: nf.dedicated_worker,
+            deployment_message: nf.deployment_message,
         },
         PushArgs::empty(),
         &authed.username,
@@ -631,25 +613,7 @@ async fn update_flow(
         .await?;
     }
 
-    tx = PushIsolationLevel::Transaction(new_tx);
-    tx = deployment_metadata_helpers::handle_deployment_metadata(
-        tx,
-        &authed,
-        &db,
-        &w_id,
-        deployment_metadata_helpers::DeployedObject::Flow { path: nf.path.clone() },
-        nf.deployment_message,
-    )
-    .await?;
-
-    match tx {
-        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
-        _ => {
-            return Err(Error::InternalErr(
-                "Expected a transaction here".to_string(),
-            ));
-        }
-    }
+    new_tx.commit().await?;
 
     Ok(nf.path.to_string())
 }

backend/windmill-api/src/job_helpers.rs

@@ -1,19 +1,19 @@
 use std::{cmp, time::Duration};
 
-use crate::{
-    db::DB, resources::get_resource_value_interpolated_internal, users::Tokened,
-    workspaces::LargeFileStorage,
-};
+use crate::{db::DB, resources::get_resource_value_interpolated_internal, users::Tokened};
+use anyhow::Context;
 use aws_sdk_s3::{
-    config::{BehaviorVersion, Credentials, Region},
     presigning::PresigningConfig,
+    primitives::ByteStream,
+    types::{CompletedMultipartUpload, CompletedPart},
 };
 use axum::{
     extract::{Path, Query},
-    routing::{get, post},
+    routing::{delete, get, post},
     Extension, Json, Router,
 };
 use hyper::http;
+use itertools::Itertools;
 use object_store::ClientConfigKey;
 use polars::{
     io::{
@@ -28,7 +28,11 @@ use polars::{
 };
 use serde::{Deserialize, Serialize};
 use tower_http::cors::{Any, CorsLayer};
-use windmill_common::{db::UserDB, error};
+use windmill_common::{
+    db::UserDB,
+    error,
+    s3_helpers::{build_s3_client, render_endpoint, LargeFileStorage, S3Resource},
+};
 
 use crate::db::ApiAuthed;
 
@@ -56,8 +60,8 @@ pub fn workspaced_service() -> Router {
             post(polars_connection_settings_v2).layer(cors.clone()),
         )
         .route(
-            "/v2/boto3_connection_settings",
-            post(boto3_connection_settings_v2).layer(cors.clone()),
+            "/v2/s3_resource_info",
+            post(s3_resource_info).layer(cors.clone()),
         )
         .route("/test_connection", get(test_connection).layer(cors.clone()))
         .route(
@@ -72,23 +76,15 @@ pub fn workspaced_service() -> Router {
             "/load_file_preview",
             get(load_file_preview).layer(cors.clone()),
         )
-}
-
-#[derive(Debug, Deserialize, Clone)]
-struct S3Resource {
-    #[serde(rename = "bucket")]
-    bucket: String,
-    region: String,
-    #[serde(rename = "endPoint")]
-    endpoint: String,
-    #[serde(rename = "useSSL")]
-    use_ssl: bool,
-    #[serde(rename = "accessKey")]
-    access_key: Option<String>,
-    #[serde(rename = "secretKey")]
-    secret_key: Option<String>,
-    #[serde(rename = "pathStyle")]
-    path_style: bool,
+        .route(
+            "/delete_s3_file",
+            delete(delete_s3_file).layer(cors.clone()),
+        )
+        .route("/move_s3_file", get(move_s3_file).layer(cors.clone()))
+        .route(
+            "/multipart_upload_s3_file",
+            post(multipart_upload_s3_file).layer(cors.clone()),
+        )
 }
 
 #[derive(Deserialize)]
@@ -201,7 +197,7 @@ struct PolarsConnectionSettingsQueryV2 {
 #[derive(Serialize)]
 struct PolarsConnectionSettingsResponse {
     s3fs_args: S3fsArgs,
-    polars_cloud_options: PolarsCloudOptions,
+    storage_options: PolarsStorageOptions,
 }
 
 #[derive(Serialize)]
@@ -217,14 +213,14 @@ struct S3fsArgs {
 }
 
 #[derive(Serialize)]
-struct PolarsCloudOptions {
+struct PolarsStorageOptions {
     aws_endpoint_url: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     aws_access_key_id: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     aws_secret_access_key: Option<String>,
     aws_region: String,
-    aws_allow_http: bool,
+    aws_allow_http: String,
 }
 
 async fn polars_connection_settings_v2(
@@ -260,39 +256,30 @@ async fn polars_connection_settings_v2(
     .0;
     let response = PolarsConnectionSettingsResponse {
         s3fs_args: s3fs,
-        polars_cloud_options: PolarsCloudOptions {
+        storage_options: PolarsStorageOptions {
             aws_endpoint_url: render_endpoint(&s3_resource),
             aws_access_key_id: s3_resource.access_key,
             aws_secret_access_key: s3_resource.secret_key,
             aws_region: s3_resource.region,
-            aws_allow_http: !s3_resource.use_ssl,
+            aws_allow_http: (!s3_resource.use_ssl).to_string(),
         },
     };
     return Ok(Json(response));
 }
 
 #[derive(Deserialize)]
-struct Boto3ConnectionSettingsQueryV2 {
+struct S3ResourceInfoQuery {
     s3_resource_path: Option<String>,
 }
 
-#[derive(Serialize)]
-struct Boto3ConnectionSettingsResponse {
-    region_name: String,
-    use_ssl: bool,
-    endpoint_url: String,
-    aws_access_key_id: Option<String>,
-    aws_secret_access_key: Option<String>,
-}
-
-async fn boto3_connection_settings_v2(
+async fn s3_resource_info(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(db): Extension<DB>,
     Tokened { token }: Tokened,
     Path(w_id): Path<String>,
-    Json(query): Json<Boto3ConnectionSettingsQueryV2>,
-) -> error::JsonResult<Boto3ConnectionSettingsResponse> {
+    Json(query): Json<S3ResourceInfoQuery>,
+) -> error::JsonResult<S3Resource> {
     let s3_resource_opt = match query.s3_resource_path {
         Some(s3_resource_path) => {
             get_s3_resource(
@@ -310,14 +297,7 @@ async fn boto3_connection_settings_v2(
     let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
         "No datasets storage resource defined at the workspace level".to_string(),
     ))?;
-    let response = Boto3ConnectionSettingsResponse {
-        endpoint_url: render_endpoint(&s3_resource),
-        region_name: s3_resource.region,
-        use_ssl: s3_resource.use_ssl,
-        aws_access_key_id: s3_resource.access_key,
-        aws_secret_access_key: s3_resource.secret_key,
-    };
-    return Ok(Json(response));
+    return Ok(Json(s3_resource));
 }
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -349,7 +329,10 @@ async fn test_connection(
         .max_keys(1)
         .send()
         .await
-        .map_err(|err| error::Error::InternalErr(err.to_string()))?;
+        .map_err(|err| {
+            tracing::error!("Error testing connection to S3 bucket: {:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
     return Ok(Json(()));
 }
 
@@ -679,6 +662,246 @@ async fn load_file_preview(
     return Ok(Json(response));
 }
 
+#[derive(Deserialize)]
+struct DeleteS3FileQuery {
+    pub file_key: String,
+}
+
+async fn delete_s3_file(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Query(query): Query<DeleteS3FileQuery>,
+) -> error::JsonResult<()> {
+    let file_key = query.file_key.clone();
+    let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
+
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3_client = build_s3_client(&s3_resource);
+
+    let s3_bucket = s3_resource.bucket.clone();
+    s3_client
+        .delete_object()
+        .bucket(&s3_bucket)
+        .key(&file_key)
+        .send()
+        .await
+        .map_err(|err| {
+            tracing::error!("{:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
+    return Ok(Json(()));
+}
+
+#[derive(Deserialize)]
+struct MoveS3FileQuery {
+    pub src_file_key: String,
+    pub dest_file_key: String,
+}
+
+async fn move_s3_file(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Query(query): Query<MoveS3FileQuery>,
+) -> error::JsonResult<()> {
+    let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
+
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let s3_client = build_s3_client(&s3_resource);
+
+    let s3_bucket = s3_resource.bucket.clone();
+    let source_uri = format!("{}/{}", s3_bucket, query.src_file_key);
+    s3_client
+        .copy_object()
+        .copy_source(&source_uri)
+        .bucket(&s3_bucket)
+        .key(&query.dest_file_key)
+        .send()
+        .await
+        .map_err(|err| {
+            tracing::error!("{:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
+
+    s3_client
+        .delete_object()
+        .bucket(&s3_bucket)
+        .key(&query.src_file_key)
+        .send()
+        .await
+        .map_err(|err| {
+            tracing::error!("{:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
+    return Ok(Json(()));
+}
+
+#[derive(Deserialize)]
+struct UploadFileQuery {
+    pub file_key: String,
+    pub part_content: Vec<u8>,
+    pub upload_id: Option<String>, // should be None for the first call to initiate the upload
+
+    pub parts: Vec<UploadFilePart>, // parts already uploaded, with their part_number and the tag associated
+
+    pub is_final: bool,      // whether it's the final chunk
+    pub cancel_upload: bool, // whether the upload should be cancelled. upload_id should be set. subsequent calls with this upload_id will fail
+}
+
+#[derive(Deserialize, Serialize, Clone)]
+struct UploadFilePart {
+    pub part_number: u16,
+    pub tag: String,
+}
+
+#[derive(Serialize)]
+struct UploadFileResponse {
+    pub upload_id: String,
+    pub parts: Vec<UploadFilePart>, // parts already uploaded, with their part_number and the tag associated
+    pub is_done: bool, // whether the transfer is finished, either b/c it got cancelled or because the last chunk was uploaded
+}
+
+async fn multipart_upload_s3_file(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Tokened { token }: Tokened,
+    Path(w_id): Path<String>,
+    Json(query): Json<UploadFileQuery>,
+) -> error::JsonResult<UploadFileResponse> {
+    tracing::debug!(
+        "Multi part file upload: part number: {} - is_final: {}",
+        query.parts.len(),
+        query.is_final
+    );
+    let file_key = query.file_key.clone();
+    let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
+
+    let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
+        "No files storage resource defined at the workspace level".to_string(),
+    ))?;
+    let bucket = s3_resource.bucket.clone();
+    let s3_client = build_s3_client(&s3_resource);
+
+    if query.cancel_upload && query.upload_id.clone().is_some() {
+        let upload_id = query.upload_id.unwrap();
+        s3_client
+            .abort_multipart_upload()
+            .bucket(&bucket)
+            .key(&file_key)
+            .upload_id(&upload_id)
+            .send()
+            .await
+            .map_err(|err| {
+                tracing::error!("{:?}", err);
+                error::Error::InternalErr(err.to_string())
+            })?;
+        return Ok(Json(UploadFileResponse {
+            upload_id: upload_id,
+            parts: vec![], // empty parts as the transfer has been cancelled
+            is_done: true,
+        }));
+    }
+
+    let (upload_id, part_number) = match query {
+        UploadFileQuery { upload_id: Some(upload_id), ref parts, .. } if parts.len() > 0 => {
+            (upload_id, parts.len() + 1)
+        }
+        UploadFileQuery { upload_id: None, ref parts, .. } if parts.len() == 0 => {
+            let multipart_upload_res = s3_client
+                .create_multipart_upload()
+                .bucket(&bucket)
+                .key(&file_key)
+                .send()
+                .await
+                .map_err(|err| {
+                    tracing::error!("{:?}", err);
+                    error::Error::InternalErr(err.to_string())
+                })?;
+            let upload_id = multipart_upload_res
+                .upload_id
+                .context("Upload ID is missing in the response")?;
+            (upload_id, 1)
+        }
+        _ => {
+            return Err(error::Error::BadRequest(
+                "parts should be empty when upload_id is not provided, as a new upload will be created."
+                    .to_string(),
+            ))
+        }
+    };
+
+    let chunk_content = query.part_content.clone();
+    let chunk_content_stream = ByteStream::from(chunk_content);
+    let multipart_upload_res = s3_client
+        .upload_part()
+        .bucket(&bucket)
+        .key(&file_key)
+        .upload_id(&upload_id)
+        .body(chunk_content_stream)
+        .part_number(part_number as i32)
+        .send()
+        .await
+        .map_err(|err| {
+            tracing::error!("{:?}", err);
+            error::Error::InternalErr(err.to_string())
+        })?;
+
+    let mut new_parts = query.parts.clone();
+    new_parts.push(UploadFilePart {
+        part_number: part_number as u16,
+        tag: multipart_upload_res.e_tag.unwrap_or_default(),
+    });
+
+    if query.is_final {
+        let completed_parts = new_parts
+            .iter()
+            .map(|part| {
+                CompletedPart::builder()
+                    .e_tag(&part.tag)
+                    .part_number(part.part_number as i32)
+                    .build()
+            })
+            .collect_vec();
+        let _complete_multipart_upload_res = s3_client
+            .complete_multipart_upload()
+            .bucket(&s3_resource.bucket)
+            .key(&query.file_key)
+            .upload_id(&upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .set_parts(Some(completed_parts))
+                    .build(),
+            )
+            .send()
+            .await
+            .map_err(|err| {
+                tracing::error!("{:?}", err);
+                error::Error::InternalErr(err.to_string())
+            })?;
+    }
+
+    return Ok(Json(UploadFileResponse {
+        upload_id: upload_id,
+        parts: new_parts,
+        is_done: query.is_final,
+    }));
+}
+
+#[derive(Deserialize)]
+pub struct S3Object {
+    pub s3: String,
+}
+
 async fn get_workspace_s3_resource<'c>(
     authed: &ApiAuthed,
     user_db: &UserDB,
@@ -751,30 +974,6 @@ async fn get_s3_resource<'c>(
     return Ok(Some(s3_resource));
 }
 
-fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
-    let s3_resource = s3_resource_ref.clone();
-
-    let endpoint = render_endpoint(&s3_resource);
-    let mut s3_config_builder = aws_sdk_s3::Config::builder()
-        .endpoint_url(endpoint)
-        .behavior_version(BehaviorVersion::latest())
-        .region(Region::new(s3_resource.region));
-    if s3_resource.access_key.is_some() {
-        s3_config_builder = s3_config_builder.credentials_provider(Credentials::new(
-            s3_resource.access_key.unwrap_or_default(),
-            s3_resource.secret_key.unwrap_or_default(),
-            None,
-            None,
-            "s3_storage",
-        ));
-    }
-    if s3_resource.path_style {
-        s3_config_builder = s3_config_builder.force_path_style(true);
-    }
-    let s3_config = s3_config_builder.build();
-    return aws_sdk_s3::Client::from_conf(s3_config);
-}
-
 fn build_polars_s3_config(s3_resource_ref: &S3Resource) -> CloudOptions {
     let s3_resource = s3_resource_ref.to_owned();
     let mut s3_configs: Vec<(AmazonS3ConfigKey, String)> = vec![
@@ -802,16 +1001,6 @@ fn build_polars_s3_config(s3_resource_ref: &S3Resource) -> CloudOptions {
     return CloudOptions::default().with_aws(s3_configs);
 }
 
-fn render_endpoint(s3_resource: &S3Resource) -> String {
-    if s3_resource.endpoint.starts_with("http://") || s3_resource.endpoint.starts_with("https://") {
-        s3_resource.endpoint.clone()
-    } else if s3_resource.use_ssl {
-        format!("https://{}", s3_resource.endpoint)
-    } else {
-        format!("http://{}", s3_resource.endpoint)
-    }
-}
-
 async fn read_s3_text_object_head(
     s3_client: &aws_sdk_s3::Client,
     s3_bucket: &str,

backend/windmill-api/src/job_metrics.rs

@@ -0,0 +1,165 @@
+use crate::db::DB;
+
+use axum::{extract::Path, routing::post, Extension, Json, Router};
+use hyper::http;
+use serde::{Deserialize, Serialize};
+use tower_http::cors::{Any, CorsLayer};
+use uuid::Uuid;
+use windmill_common::{
+    error,
+    job_metrics::{JobStatsRecord, MetricKind},
+};
+
+pub fn workspaced_service() -> Router {
+    let cors = CorsLayer::new()
+        .allow_methods([http::Method::GET, http::Method::POST])
+        .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
+        .allow_origin(Any);
+
+    Router::new().route("/get/:id", post(get_job_metrics).layer(cors.clone()))
+}
+
+#[derive(Deserialize)]
+struct JobStatsRequest {
+    from_timestamp: Option<chrono::DateTime<chrono::Utc>>,
+    to_timestamp: Option<chrono::DateTime<chrono::Utc>>,
+    timeseries_max_datapoints: Option<u32>, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all
+}
+
+#[derive(Serialize)]
+struct JobStatsResponse {
+    metrics_metadata: Vec<MetricsMetadata>,
+    scalar_metrics: Vec<ScalarMetric>,
+    timeseries_metrics: Vec<TimeseriesMetric>,
+}
+
+#[derive(Serialize)]
+pub struct MetricsMetadata {
+    pub id: String,
+    pub name: Option<String>,
+}
+
+#[derive(Serialize)]
+struct ScalarMetric {
+    metric_id: String,
+    value: f64,
+}
+
+#[derive(Serialize)]
+struct TimeseriesMetric {
+    metric_id: String,
+    values: Vec<DataPoint>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct DataPoint {
+    pub timestamp: chrono::DateTime<chrono::Utc>,
+    pub value: f64,
+}
+
+async fn get_job_metrics(
+    Extension(db): Extension<DB>,
+    Path((w_id, job_id)): Path<(String, Uuid)>,
+    Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json<
+        JobStatsRequest,
+    >,
+) -> error::JsonResult<JobStatsResponse> {
+    let records = sqlx::query_as::<_, JobStatsRecord>(
+        "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2",
+    )
+    .bind(w_id)
+    .bind(job_id)
+    .fetch_all(&db)
+    .await?;
+
+    let mut metrics_metadata: Vec<MetricsMetadata> = vec![];
+    let mut scalar_metrics: Vec<ScalarMetric> = vec![];
+    let mut timeseries_metrics: Vec<TimeseriesMetric> = vec![];
+
+    for record in records {
+        let metric_id = record.metric_id;
+        match record.metric_kind {
+            MetricKind::ScalarInt => {
+                let value = record.scalar_int.unwrap_or_default() as f64;
+                scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
+            }
+            MetricKind::ScalarFloat => {
+                let value = record.scalar_float.unwrap_or_default() as f64;
+                scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
+            }
+            MetricKind::TimeseriesInt => {
+                if record.timestamps.clone().unwrap_or_default().len()
+                    != record.timeseries_int.clone().unwrap_or_default().len()
+                {
+                    tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int)
+                }
+                let (timestamps, timeseries_int) = timeseries_sample(
+                    from_timestamp,
+                    to_timestamp,
+                    timeseries_max_datapoints,
+                    record.timestamps.unwrap_or_default(),
+                    record.timeseries_int.unwrap_or_default(),
+                );
+                let mut values: Vec<DataPoint> = vec![];
+                for (idx, value) in timeseries_int.iter().enumerate() {
+                    values.push(DataPoint {
+                        timestamp: timestamps[idx],
+                        value: value.to_owned() as f64,
+                    });
+                }
+                timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
+            }
+            MetricKind::TimeseriesFloat => {
+                if record.timestamps.clone().unwrap_or_default().len()
+                    != record.timeseries_int.clone().unwrap_or_default().len()
+                {
+                    tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float)
+                }
+                let (timestamps, timeseries_float) = timeseries_sample(
+                    from_timestamp,
+                    to_timestamp,
+                    timeseries_max_datapoints,
+                    record.timestamps.unwrap_or_default(),
+                    record.timeseries_float.unwrap_or_default(),
+                );
+                let mut values: Vec<DataPoint> = vec![];
+                for (idx, value) in timeseries_float.iter().enumerate() {
+                    values.push(DataPoint {
+                        timestamp: timestamps[idx],
+                        value: value.to_owned() as f64,
+                    });
+                }
+                timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
+            }
+        };
+        metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name });
+    }
+
+    let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics };
+    Ok(Json(response))
+}
+
+fn timeseries_sample<T: Copy>(
+    from: Option<chrono::DateTime<chrono::Utc>>,
+    to: Option<chrono::DateTime<chrono::Utc>>,
+    _datapoints: Option<u32>,
+    timestamps: Vec<chrono::DateTime<chrono::Utc>>,
+    values: Vec<T>,
+) -> (Vec<chrono::DateTime<chrono::Utc>>, Vec<T>) {
+    if timestamps.len() != values.len() {
+        tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len());
+        return (vec![], vec![]);
+    }
+    let mut filtered_timestamp: Vec<chrono::DateTime<chrono::Utc>> = vec![];
+    let mut filtered_values: Vec<T> = vec![];
+    for (idx, timestamp) in timestamps.iter().enumerate() {
+        if *timestamp > from.unwrap_or(chrono::DateTime::<chrono::Utc>::MIN_UTC)
+            && *timestamp < to.unwrap_or(chrono::DateTime::<chrono::Utc>::MAX_UTC)
+        {
+            filtered_timestamp.push(timestamps[idx]);
+            filtered_values.push(values[idx]);
+        }
+    }
+    // TODO: implement sampling
+    return (filtered_timestamp, filtered_values);
+}

backend/windmill-api/src/jobs.rs

@@ -2826,14 +2826,14 @@ pub struct JobUpdate {
 
 async fn get_job_update(
     Extension(db): Extension<DB>,
-    Path((w_id, id)): Path<(String, Uuid)>,
+    Path((w_id, job_id)): Path<(String, Uuid)>,
     Query(JobUpdateQuery { running, log_offset }): Query<JobUpdateQuery>,
 ) -> error::JsonResult<JobUpdate> {
     let record = sqlx::query!(
         "SELECT running, substr(logs, $1) as logs, mem_peak FROM queue WHERE workspace_id = $2 AND id = $3",
         log_offset,
         &w_id,
-        &id
+        &job_id
     )
     .fetch_optional(&db)
     .await?;
@@ -2855,11 +2855,11 @@ async fn get_job_update(
              $3",
             log_offset,
             &w_id,
-            &id
+            &job_id
         )
         .fetch_optional(&db)
         .await?;
-        let logs = not_found_if_none(logs, "Job Update", id.to_string())?;
+        let logs = not_found_if_none(logs, "Job Update", job_id.to_string())?;
         Ok(Json(JobUpdate {
             running: Some(false),
             completed: Some(true),

backend/windmill-api/src/lib.rs

@@ -49,7 +49,6 @@ mod audit;
 mod capture;
 mod configs;
 mod db;
-mod deployment_metadata_helpers;
 mod drafts;
 pub mod ee;
 pub mod embeddings;
@@ -61,6 +60,7 @@ mod groups;
 mod inputs;
 mod integration;
 pub mod job_helpers;
+pub mod job_metrics;
 pub mod jobs;
 pub mod oauth2;
 mod openai;
@@ -198,6 +198,7 @@ pub async fn run_server(
                         .nest("/folders", folders::workspaced_service())
                         .nest("/groups", groups::workspaced_service())
                         .nest("/inputs", inputs::workspaced_service())
+                        .nest("/job_metrics", job_metrics::workspaced_service())
                         .nest("/job_helpers", job_helpers::workspaced_service())
                         .nest("/jobs", jobs::workspaced_service())
                         .nest("/oauth", oauth2::workspaced_service())

backend/windmill-api/src/schedule.rs

@@ -54,6 +54,8 @@ pub struct NewSchedule {
     pub path: String,
     pub schedule: String,
     pub timezone: String,
+    pub summary: Option<String>,
+    pub no_flow_overlap: Option<bool>,
     pub script_path: String,
     pub is_flow: bool,
     pub args: Option<serde_json::Value>,
@@ -66,6 +68,7 @@ pub struct NewSchedule {
     pub on_recovery_times: Option<i32>,
     pub on_recovery_extra_args: Option<serde_json::Value>,
     pub ws_error_handler_muted: Option<bool>,
+    pub retry: Option<serde_json::Value>,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -150,7 +153,12 @@ async fn create_schedule(
     let schedule = sqlx::query_as!(
         Schedule,
         "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
-         is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING *",
+            is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, \
+            on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, \
+            ws_error_handler_muted, retry, summary, no_flow_overlap \
+        ) VALUES ( \
+            $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21 \
+        ) RETURNING *",
         w_id,
         ns.path,
         ns.schedule,
@@ -169,6 +177,9 @@ async fn create_schedule(
         ns.on_recovery_times,
         ns.on_recovery_extra_args,
         ns.ws_error_handler_muted.unwrap_or(false),
+        ns.retry,
+        ns.summary,
+        ns.no_flow_overlap.unwrap_or(false),
     )
     .fetch_one(&mut tx)
     .await
@@ -220,8 +231,10 @@ async fn edit_schedule(
     clear_schedule(tx.transaction_mut(), path, &w_id).await?;
     let schedule = sqlx::query_as!(
         Schedule,
-        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11 
-        WHERE path  = $12 AND workspace_id = $13 RETURNING *",
+        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, \
+            on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, \
+            on_recovery_extra_args = $10, ws_error_handler_muted = $11, retry = $12, summary = $13, no_flow_overlap = $14 \
+        WHERE path = $15 AND workspace_id = $16 RETURNING *",
         es.schedule,
         es.timezone,
         es.args,
@@ -233,6 +246,9 @@ async fn edit_schedule(
         es.on_recovery_times,
         es.on_recovery_extra_args,
         es.ws_error_handler_muted.unwrap_or(false),
+        es.retry,
+        es.summary,
+        es.no_flow_overlap.unwrap_or(false),
         path,
         w_id,
     )
@@ -324,7 +340,10 @@ pub struct ScheduleWJobs {
     pub on_recovery_times: Option<i32>,
     pub on_recovery_extra_args: Option<serde_json::Value>,
     pub ws_error_handler_muted: bool,
+    pub retry: Option<serde_json::Value>,
     pub jobs: Option<Vec<serde_json::Value>>,
+    pub summary: Option<String>,
+    pub no_flow_overlap: bool,
 }
 
 async fn list_schedule_with_jobs(
@@ -337,7 +356,7 @@ async fn list_schedule_with_jobs(
     let (per_page, offset) = paginate(pagination);
     let rows = sqlx::query_as!(ScheduleWJobs,
         "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE
-        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t
+        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL AND is_skipped = False ORDER BY started_at DESC LIMIT 20) AS jobs ) t
         WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
         w_id,
         per_page as i64,
@@ -662,6 +681,7 @@ pub struct EditSchedule {
     pub schedule: String,
     pub timezone: String,
     pub args: Option<serde_json::Value>,
+    pub summary: Option<String>,
     pub on_failure: Option<String>,
     pub on_failure_times: Option<i32>,
     pub on_failure_exact: Option<bool>,
@@ -670,6 +690,8 @@ pub struct EditSchedule {
     pub on_recovery_times: Option<i32>,
     pub on_recovery_extra_args: Option<serde_json::Value>,
     pub ws_error_handler_muted: Option<bool>,
+    pub retry: Option<serde_json::Value>,
+    pub no_flow_overlap: Option<bool>,
 }
 
 pub async fn clear_schedule<'c>(

backend/windmill-api/src/scripts.rs

@@ -8,7 +8,6 @@
 
 use crate::{
     db::{ApiAuthed, DB},
-    deployment_metadata_helpers,
     schedule::clear_schedule,
     users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
     webhook_util::{WebhookMessage, WebhookShared},
@@ -47,6 +46,7 @@ use windmill_common::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
 };
+use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
 use windmill_queue::{
     self, schedule::push_scheduled_job, PushArgs, PushIsolationLevel, QueueTransaction,
 };
@@ -325,7 +325,7 @@ async fn create_script(
     let script_path = ns.path.clone();
     let hash = ScriptHash(hash_script(&ns));
     let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
+    let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), user_db.begin(&authed).await?).into();
 
     if sqlx::query_scalar!(
         "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -604,37 +604,29 @@ async fn create_script(
         );
     }
 
-    let mut tx = PushIsolationLevel::Transaction(tx);
-
+    let permissioned_as = username_to_permissioned_as(&authed.username);
     if needs_lock_gen {
-        let dependencies = match ns.language {
-            ScriptLang::Python3 => {
-                windmill_parser_py_imports::parse_python_imports(&ns.content, &w_id, &ns.path, &db)
-                    .await?
-                    .join("\n")
-            }
-            _ => ns.content,
-        };
         let tag = if ns.dedicated_worker.is_some_and(|x| x) {
             Some(format!("{}:{}", &w_id, &ns.path,))
         } else {
             ns.tag
         };
+        let tx = PushIsolationLevel::Transaction(tx);
         let (_, new_tx) = windmill_queue::push(
             &db,
             tx,
             &w_id,
             JobPayload::Dependencies {
                 hash,
-                dependencies,
                 language: ns.language,
                 path: ns.path,
                 dedicated_worker: ns.dedicated_worker,
+                deployment_message: ns.deployment_message,
             },
             PushArgs::empty(),
             &authed.username,
             &authed.email,
-            username_to_permissioned_as(&authed.username),
+            permissioned_as,
             None,
             None,
             None,
@@ -650,26 +642,19 @@ async fn create_script(
             None,
         )
         .await?;
-        tx = PushIsolationLevel::Transaction(new_tx);
-    }
-
-    tx = deployment_metadata_helpers::handle_deployment_metadata(
-        tx,
-        &authed,
-        &db,
-        &w_id,
-        deployment_metadata_helpers::DeployedObject::Script { hash: hash, path: script_path },
-        ns.deployment_message,
-    )
-    .await?;
-
-    match tx {
-        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
-        _ => {
-            return Err(Error::InternalErr(
-                "Expected a transaction here".to_string(),
-            ));
-        }
+        new_tx.commit().await?;
+    } else {
+        handle_deployment_metadata(
+            &authed.email,
+            &authed.username,
+            &db,
+            &w_id,
+            DeployedObject::Script { hash: hash, path: script_path },
+            ns.deployment_message,
+            rsmq,
+        )
+        .await?;
+        tx.commit().await?;
     }
 
     Ok((StatusCode::CREATED, format!("{}", hash)))

backend/windmill-api/src/workspaces.rs

@@ -40,9 +40,11 @@ use stripe::CustomerId;
 use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::db::UserDB;
+use windmill_common::s3_helpers::LargeFileStorage;
 use windmill_common::schedule::Schedule;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::worker::CLOUD_HOSTED;
+use windmill_common::workspaces::WorkspaceGitRepo;
 use windmill_common::{
     error::{to_anyhow, Error, JsonResult, Result},
     flows::Flow,
@@ -281,18 +283,6 @@ pub struct EditErrorHandler {
     pub error_handler_muted_on_cancel: Option<bool>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(tag = "type")]
-pub enum LargeFileStorage {
-    S3Storage(S3Storage),
-    // TODO: Add a filesystem type here in the future if needed
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct S3Storage {
-    pub s3_resource_path: String,
-}
-
 async fn list_pending_invites(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -1056,12 +1046,6 @@ struct EditGitSyncConfig {
     git_sync_settings: Option<WorkspaceGitRepo>,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct WorkspaceGitRepo {
-    pub script_path: String,
-    pub git_repo_resource_path: String,
-}
-
 async fn edit_git_sync_config(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -1515,6 +1499,9 @@ async fn delete_workspace(
     let mut tx = db.begin().await?;
     require_super_admin(&db, &email).await?;
 
+    sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
     sqlx::query!("DELETE FROM queue WHERE workspace_id = $1", &w_id)
         .execute(&mut *tx)
         .await?;
@@ -1533,6 +1520,12 @@ async fn delete_workspace(
     sqlx::query!("DELETE FROM app WHERE workspace_id = $1", &w_id)
         .execute(&mut *tx)
         .await?;
+    sqlx::query!("DELETE FROM raw_app WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+    sqlx::query!("DELETE FROM input WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
     sqlx::query!("DELETE FROM variable WHERE workspace_id = $1", &w_id)
         .execute(&mut *tx)
         .await?;
@@ -1548,6 +1541,17 @@ async fn delete_workspace(
         .execute(&mut *tx)
         .await?;
 
+    sqlx::query!("DELETE FROM job_stats WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!(
+        "DELETE FROM deployment_metadata WHERE workspace_id = $1",
+        &w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
     sqlx::query!("DELETE FROM usr WHERE workspace_id = $1", &w_id)
         .execute(&mut *tx)
         .await?;
@@ -1575,6 +1579,10 @@ async fn delete_workspace(
         .execute(&mut *tx)
         .await?;
 
+    sqlx::query!("DELETE FROM account WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
     sqlx::query!("DELETE FROM workspace_key WHERE workspace_id = $1", &w_id)
         .execute(&mut *tx)
         .await?;
@@ -1586,6 +1594,10 @@ async fn delete_workspace(
     .execute(&mut *tx)
     .await?;
 
+    sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
     sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id)
         .execute(&mut *tx)
         .await?;

backend/windmill-audit/src/lib.rs

@@ -152,11 +152,17 @@ pub async fn list_audit(
     Ok(rows)
 }
 
-pub async fn get_audit(mut tx: Transaction<'_, sqlx::Postgres>, id: i32) -> Result<AuditLog> {
-    let audit_o = sqlx::query_as::<_, AuditLog>("SELECT * FROM audit WHERE id = $1")
-        .bind(id)
-        .fetch_optional(&mut *tx)
-        .await?;
+pub async fn get_audit(
+    mut tx: Transaction<'_, sqlx::Postgres>,
+    id: i32,
+    w_id: &str,
+) -> Result<AuditLog> {
+    let audit_o =
+        sqlx::query_as::<_, AuditLog>("SELECT * FROM audit WHERE id = $1 AND workspace_id = $2")
+            .bind(id)
+            .bind(w_id)
+            .fetch_optional(&mut *tx)
+            .await?;
     tx.commit().await?;
     let audit = windmill_common::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?;
     Ok(audit)

backend/windmill-common/Cargo.toml

@@ -48,4 +48,6 @@ itertools.workspace = true
 regex.workspace = true
 git-version.workspace = true
 cron.workspace = true
-tracing-loki = { version = "^0", optional = true }
+tracing-loki = { version = "^0", optional = true }
+aws-sdk-s3.workspace = true
+aws-config.workspace = true

backend/windmill-common/src/job_metrics.rs

@@ -0,0 +1,187 @@
+use crate::{db::DB, error};
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(sqlx::FromRow)]
+pub struct JobStatsRecord {
+    pub workspace_id: String,
+    pub job_id: Uuid,
+    pub metric_id: String,
+    pub metric_name: Option<String>,
+    pub metric_kind: MetricKind,
+    pub scalar_int: Option<i32>,
+    pub scalar_float: Option<f32>,
+    pub timestamps: Option<Vec<chrono::DateTime<chrono::Utc>>>,
+    pub timeseries_int: Option<Vec<i32>>,
+    pub timeseries_float: Option<Vec<f32>>,
+}
+
+#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)]
+#[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")]
+pub enum MetricKind {
+    ScalarInt,
+    ScalarFloat,
+    TimeseriesInt,
+    TimeseriesFloat,
+}
+
+pub enum MetricNumericValue {
+    Integer(i32),
+    Float(f32),
+}
+
+pub async fn register_metric_for_job(
+    db: &DB,
+    workspace_id: String,
+    job_id: Uuid,
+    metric_id: String,
+    metric_kind: MetricKind,
+    metric_name: Option<String>,
+) -> error::Result<String> {
+    let exists = sqlx::query_scalar!(
+        "SELECT true FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+        workspace_id,
+        job_id,
+        metric_id
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten();
+    if exists.unwrap_or(false) {
+        return Err(error::Error::BadRequest(format!(
+            "Metric {} is already registered for job {}",
+            metric_id, job_id
+        )));
+    }
+
+    let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind
+    {
+        MetricKind::ScalarInt | MetricKind::ScalarFloat => {
+            (None as Option<i32>, None as Option<f32>, None, None, None)
+        }
+        MetricKind::TimeseriesInt => (
+            None,
+            None,
+            Some(&[] as &[chrono::DateTime<chrono::Utc>]),
+            Some(&[] as &[i32]),
+            None,
+        ),
+        MetricKind::TimeseriesFloat => (
+            None,
+            None,
+            Some(&[] as &[chrono::DateTime<chrono::Utc>]),
+            None,
+            Some(&[] as &[f32]),
+        ),
+    };
+
+    sqlx::query(
+        "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+    )
+    .bind(workspace_id)
+    .bind(job_id)
+    .bind(&metric_id)
+    .bind(metric_name)
+    .bind(metric_kind)
+    .bind(scalar_int)
+    .bind(scalar_float)
+    .bind(timestamps)
+    .bind(timeseries_int)
+    .bind(timeseries_float)
+    .execute(db)
+    .await?;
+
+    Ok(metric_id)
+}
+
+pub async fn record_metric(
+    db: &DB,
+    workspace_id: String,
+    job_id: Uuid,
+    metric_id: String,
+    value: MetricNumericValue,
+) -> error::Result<()> {
+    let metric_kind_opt: Option<MetricKind> = sqlx::query_scalar(
+        "SELECT metric_kind FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+    )
+    .bind(&workspace_id)
+    .bind(&job_id)
+    .bind(&metric_id)
+    .fetch_optional(db)
+    .await?;
+
+    if metric_kind_opt.is_none() {
+        return Err(error::Error::BadRequest(format!(
+            "Metric {} not yet registered for job {}.",
+            metric_id, job_id
+        )));
+    }
+    let metric_kind = metric_kind_opt.unwrap();
+
+    let (value_int, value_float) = match value {
+        MetricNumericValue::Integer(val) => {
+            if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt {
+                return Err(error::Error::BadRequest(format!(
+                    "Metric {} is not a timeseries int metric.",
+                    metric_id
+                )));
+            }
+            (val, 0 as f32)
+        }
+        MetricNumericValue::Float(val) => {
+            if metric_kind != MetricKind::TimeseriesFloat && metric_kind != MetricKind::ScalarFloat
+            {
+                return Err(error::Error::BadRequest(format!(
+                    "Metric {} is not a timeseries float metric.",
+                    metric_id
+                )));
+            }
+            (0 as i32, val)
+        }
+    };
+
+    let timestamp = chrono::Utc::now();
+    match metric_kind {
+        MetricKind::ScalarInt => {
+            sqlx::query!(
+                "UPDATE job_stats SET scalar_int = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+                &workspace_id,
+                &job_id,
+                &metric_id,
+                value_int,
+            ).execute(db).await?;
+        }
+        MetricKind::ScalarFloat => {
+            sqlx::query!(
+                "UPDATE job_stats SET scalar_float = $4 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+                &workspace_id,
+                &job_id,
+                &metric_id,
+                value_float,
+            ).execute(db).await?;
+        }
+        MetricKind::TimeseriesInt => {
+            sqlx::query!(
+                "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_int = timeseries_int || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+                &workspace_id,
+                &job_id,
+                &metric_id,
+                &[timestamp],
+                &[value_int]
+            ).execute(db).await?;
+        }
+        MetricKind::TimeseriesFloat => {
+            sqlx::query!(
+                "UPDATE job_stats SET timestamps = timestamps || $4, timeseries_float = timeseries_float || $5 WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
+                &workspace_id,
+                &job_id,
+                &metric_id,
+                &[timestamp],
+                &[value_float]
+            ).execute(db).await?;
+        }
+    }
+
+    Ok(())
+}

backend/windmill-common/src/jobs.rs

@@ -8,7 +8,7 @@ use uuid::Uuid;
 use crate::{
     error::{self, Error},
     flow_status::{FlowStatus, RestartedFrom},
-    flows::FlowValue,
+    flows::{FlowValue, Retry},
     get_latest_deployed_hash_for_path,
     scripts::{ScriptHash, ScriptLang},
 };
@@ -24,6 +24,7 @@ pub enum JobKind {
     Dependencies,
     Flow,
     FlowPreview,
+    SingleScriptFlow,
     Identity,
     FlowDependencies,
     AppDependencies,
@@ -282,17 +283,19 @@ pub enum JobPayload {
     Dependencies {
         path: String,
         hash: ScriptHash,
-        dependencies: String,
         language: ScriptLang,
         dedicated_worker: Option<bool>,
+        deployment_message: Option<String>,
     },
     FlowDependencies {
         path: String,
         dedicated_worker: Option<bool>,
+        deployment_message: Option<String>,
     },
     AppDependencies {
         path: String,
         version: i64,
+        deployment_message: Option<String>,
     },
     Flow {
         path: String,
@@ -308,6 +311,16 @@ pub enum JobPayload {
         path: Option<String>,
         restarted_from: Option<RestartedFrom>,
     },
+    SingleScriptFlow {
+        path: String,
+        hash: ScriptHash,
+        args: HashMap<String, serde_json::Value>,
+        retry: Retry, // for now only used to retry the script, so retry is necessarily present
+        concurrent_limit: Option<i32>,
+        concurrency_time_window_s: Option<i32>,
+        cache_ttl: Option<i32>,
+        priority: Option<i16>,
+    },
     DeploymentCallback {
         path: String,
     },

backend/windmill-common/src/lib.rs

@@ -23,9 +23,11 @@ pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
 pub mod global_settings;
+pub mod job_metrics;
 pub mod jobs;
 pub mod more_serde;
 pub mod oauth2;
+pub mod s3_helpers;
 pub mod schedule;
 pub mod scripts;
 pub mod server;
@@ -34,6 +36,7 @@ pub mod users;
 pub mod utils;
 pub mod variables;
 pub mod worker;
+pub mod workspaces;
 
 #[cfg(feature = "tracing_init")]
 pub mod tracing_init;

backend/windmill-common/src/s3_helpers.rs

@@ -0,0 +1,96 @@
+use aws_config::{BehaviorVersion, Region};
+use aws_sdk_s3::config::Credentials;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(tag = "type")]
+pub enum LargeFileStorage {
+    S3Storage(S3Storage),
+    // TODO: Add a filesystem type here in the future if needed
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct S3Storage {
+    pub s3_resource_path: String,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct S3Resource {
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+    pub region: String,
+    #[serde(rename = "endPoint")]
+    pub endpoint: String,
+    #[serde(rename = "useSSL")]
+    pub use_ssl: bool,
+    #[serde(rename = "accessKey")]
+    pub access_key: Option<String>,
+    #[serde(rename = "secretKey")]
+    pub secret_key: Option<String>,
+    #[serde(rename = "pathStyle")]
+    pub path_style: bool,
+    pub port: Option<u16>,
+}
+
+#[derive(Deserialize, Clone)]
+pub struct S3Object {
+    pub s3: String,
+}
+
+pub async fn get_etag_or_empty(s3_resource: &S3Resource, s3_object: S3Object) -> Option<String> {
+    let s3_client = build_s3_client(s3_resource);
+
+    let s3_bucket = s3_resource.bucket.clone();
+    let s3_key = s3_object.s3;
+    let s3_object_metadata = s3_client
+        .head_object()
+        .bucket(s3_bucket)
+        .key(s3_key)
+        .send()
+        .await;
+    s3_object_metadata
+        .ok()
+        .map(|res| res.e_tag().map(|et| et.to_string()))
+        .flatten()
+}
+
+pub fn render_endpoint(s3_resource: &S3Resource) -> String {
+    let url_with_prefix = if s3_resource.endpoint.starts_with("http://")
+        || s3_resource.endpoint.starts_with("https://")
+    {
+        s3_resource.endpoint.clone()
+    } else if s3_resource.use_ssl {
+        format!("https://{}", s3_resource.endpoint)
+    } else {
+        format!("http://{}", s3_resource.endpoint)
+    };
+    if s3_resource.port.is_some() {
+        format!("{}:{}", url_with_prefix, s3_resource.port.unwrap())
+    } else {
+        url_with_prefix
+    }
+}
+
+pub fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
+    let s3_resource = s3_resource_ref.clone();
+
+    let endpoint = render_endpoint(&s3_resource);
+    let mut s3_config_builder = aws_sdk_s3::Config::builder()
+        .endpoint_url(endpoint)
+        .behavior_version(BehaviorVersion::latest())
+        .region(Region::new(s3_resource.region));
+    if s3_resource.access_key.is_some() {
+        s3_config_builder = s3_config_builder.credentials_provider(Credentials::new(
+            s3_resource.access_key.unwrap_or_default(),
+            s3_resource.secret_key.unwrap_or_default(),
+            None,
+            None,
+            "s3_storage",
+        ));
+    }
+    if s3_resource.path_style {
+        s3_config_builder = s3_config_builder.force_path_style(true);
+    }
+    let s3_config = s3_config_builder.build();
+    return aws_sdk_s3::Client::from_conf(s3_config);
+}

backend/windmill-common/src/schedule.rs

@@ -10,7 +10,9 @@ use chrono::DateTime;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 
-#[derive(FromRow, Serialize, Deserialize, Debug)]
+use crate::flows::Retry;
+
+#[derive(FromRow, Serialize, Deserialize, Debug, Clone)]
 pub struct Schedule {
     pub workspace_id: String,
     pub path: String,
@@ -34,6 +36,15 @@ pub struct Schedule {
     pub on_recovery_times: Option<i32>,
     pub on_recovery_extra_args: Option<serde_json::Value>,
     pub ws_error_handler_muted: bool,
+    pub retry: Option<serde_json::Value>,
+    pub no_flow_overlap: bool,
+    pub summary: Option<String>,
+}
+
+impl Schedule {
+    pub fn parse_retry(self) -> Option<Retry> {
+        self.retry.map(|r| serde_json::from_value(r).ok()).flatten()
+    }
 }
 
 pub fn schedule_to_user(path: &str) -> String {

backend/windmill-common/src/users.rs

@@ -8,6 +8,7 @@
 
 pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
 pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev";
+pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev";
 
 pub fn username_to_permissioned_as(user: &str) -> String {
     if user.contains('@') {

backend/windmill-common/src/workspaces.rs

@@ -0,0 +1,8 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct WorkspaceGitRepo {
+    pub script_path: String,
+    pub git_repo_resource_path: String,
+    pub use_individual_branch: Option<bool>,
+}

backend/windmill-git-sync/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "windmill-git-sync"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[lib]
+name = "windmill_git_sync"
+path = "./src/lib.rs"
+
+[features]
+enterprise = ["windmill-queue/enterprise"]
+default = []
+
+[dependencies]
+serde.workspace = true
+sqlx.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+windmill-common = { workspace = true, features = ["axum"] }
+windmill-queue.workspace = true
+rsmq_async.workspace = true

backend/windmill-git-sync/src/lib.rs

@@ -8,17 +8,18 @@
 
 use std::collections::HashMap;
 
-use crate::db::ApiAuthed;
-use crate::db::DB;
-use crate::workspaces::{WorkspaceGitRepo, WorkspaceSettings};
+use sqlx::{Pool, Postgres};
+use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
+use windmill_common::workspaces::WorkspaceGitRepo;
 
 use serde_json::json;
 use windmill_common::error::{Error, Result};
 use windmill_common::jobs::JobPayload;
 use windmill_common::scripts::ScriptHash;
-use windmill_common::users::username_to_permissioned_as;
 use windmill_queue::PushIsolationLevel;
 
+pub type DB = Pool<Postgres>;
+
 #[derive(Clone)]
 pub enum DeployedObject {
     Script { hash: ScriptHash, path: String },
@@ -37,13 +38,14 @@ impl DeployedObject {
 }
 
 pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
-    mut tx: PushIsolationLevel<'c, R>,
-    authed: &ApiAuthed,
+    email: &str,
+    created_by: &str,
     db: &DB,
     w_id: &str,
     obj: DeployedObject,
     deployment_message: Option<String>,
-) -> Result<PushIsolationLevel<'c, R>> {
+    rsmq: Option<R>,
+) -> Result<()> {
     let skip_git_sync = if obj.get_path().starts_with("u/") {
         tracing::debug!(
             "Ignoring {} from git sync as it's in a private user folder",
@@ -54,10 +56,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
         false
     };
 
-    let workspace_git_repo_setting = sqlx::query_as::<_, WorkspaceSettings>(
-        "SELECT * FROM workspace_settings WHERE workspace_id = $1",
+    let workspace_git_repo_setting = sqlx::query_scalar!(
+        "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1",
+        w_id
     )
-    .bind(&w_id)
     .fetch_optional(db)
     .await?;
 
@@ -69,11 +71,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
 
     let workspace_git_repo = workspace_git_repo_setting
         .unwrap()
-        .git_sync
         .map(|conf| serde_json::from_value::<WorkspaceGitRepo>(conf).ok())
         .flatten();
 
-    let (git_sync_job_uuid, mut new_tx) = if !skip_git_sync && workspace_git_repo.is_some() {
+    let git_sync_job_uuid = if !skip_git_sync && workspace_git_repo.is_some() {
         let workspace_git_repo = workspace_git_repo.unwrap();
 
         let mut args: HashMap<String, serde_json::Value> = HashMap::new();
@@ -84,35 +85,56 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
                 .strip_prefix("$res:")),
         );
 
-        let default_commit_msg: String;
-        match obj.clone() {
+        let message = match obj.clone() {
             DeployedObject::Script { path, .. } => {
                 args.insert("path".to_string(), json!(path.to_string()));
-                default_commit_msg = format!("Script '{}' deployed", path);
+                if deployment_message.as_ref().is_none()
+                    || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+                {
+                    format!("Script '{}' deployed", path)
+                } else {
+                    deployment_message.clone().unwrap()
+                }
             }
             DeployedObject::Flow { path } => {
                 args.insert("path".to_string(), json!(path.to_string()));
-                default_commit_msg = format!("Flow '{}' deployed", path);
+                if deployment_message.as_ref().is_none()
+                    || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+                {
+                    format!("Flow '{}' deployed", path)
+                } else {
+                    deployment_message.clone().unwrap()
+                }
             }
             DeployedObject::App { path, .. } => {
                 args.insert("path".to_string(), json!(path.to_string()));
-                default_commit_msg = format!("App '{}' deployed", path);
+                if deployment_message.as_ref().is_none()
+                    || deployment_message.as_ref().is_some_and(|x| x.is_empty())
+                {
+                    format!("App '{}' deployed", path)
+                } else {
+                    deployment_message.clone().unwrap()
+                }
             }
-        }
+        };
+
+        args.insert("commit_msg".to_string(), json!(message));
         args.insert(
-            "commit_msg".to_string(),
-            json!(deployment_message.clone().unwrap_or(default_commit_msg)),
+            "use_individual_branch".to_string(),
+            json!(workspace_git_repo.use_individual_branch.unwrap_or(false)),
         );
 
+        let tx: PushIsolationLevel<'_, R> = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
+
         let (job_uuid, new_tx) = windmill_queue::push(
-            &db,
+            db,
             tx,
             &w_id,
             JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
             args,
-            &authed.username,
-            &authed.email,
-            username_to_permissioned_as(&authed.username),
+            created_by,
+            &email,
+            SUPERADMIN_SYNC_EMAIL.to_string(),
             None,
             None,
             None,
@@ -128,16 +150,10 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
             None,
         )
         .await?;
-        (Some(job_uuid), new_tx)
+        new_tx.commit().await?;
+        Some(job_uuid)
     } else {
-        let new_tx = match tx {
-            PushIsolationLevel::Isolated(user_db, authed, rsmq) => {
-                (rsmq, user_db.begin(&authed).await?).into()
-            }
-            PushIsolationLevel::IsolatedRoot(db, rsmq) => (rsmq, db.begin().await?).into(),
-            PushIsolationLevel::Transaction(tx) => tx,
-        };
-        (None, new_tx)
+        None
     };
 
     // We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
@@ -149,28 +165,27 @@ pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send
     if deployment_message.is_some() || job_uuids.len() > 0 {
         // if the git sync job hasn't been triggered, and there is not custom deployment message, there's not point adding an entry to the table
         match obj.clone() {
-            DeployedObject::Script { path, hash, .. } => {
-                sqlx::query!(
-                    "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
-                    w_id, path, hash.0, &job_uuids, deployment_message,
-                )
-            },
-            DeployedObject::Flow { path } => {
-                sqlx::query!(
-                    "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
-                    w_id, path, &job_uuids, deployment_message,
-                )
-            }
-            DeployedObject::App { path, version } => {
-                sqlx::query!(
-                    "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
-                    w_id, path, version, &job_uuids, deployment_message,
-                )
-            }
-        }.execute(&mut new_tx)
-        .await?;
+             DeployedObject::Script { path, hash, .. } => {
+                 sqlx::query!(
+                     "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+                     w_id, path, hash.0, &job_uuids, deployment_message,
+                 )
+             },
+             DeployedObject::Flow { path } => {
+                 sqlx::query!(
+                     "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3, deployment_msg = $4",
+                     w_id, path, &job_uuids, deployment_message,
+                 )
+             }
+             DeployedObject::App { path, version } => {
+                 sqlx::query!(
+                     "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)",
+                     w_id, path, version, &job_uuids, deployment_message,
+                 )
+             }
+         }.execute(db)
+         .await?;
     }
 
-    tx = PushIsolationLevel::Transaction(new_tx);
-    return Ok(tx);
+    return Ok(());
 }

backend/windmill-queue/src/jobs.rs

@@ -49,7 +49,9 @@ use windmill_common::{
         BranchAllStatus, FlowCleanupModule, FlowStatus, FlowStatusModule, FlowStatusModuleWParent,
         Iterator, JobResult, RestartedFrom, RetryStatus, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL,
     },
-    flows::{add_virtual_items_if_necessary, FlowModuleValue, FlowValue},
+    flows::{
+        add_virtual_items_if_necessary, FlowModule, FlowModuleValue, FlowValue, InputTransform,
+    },
     jobs::{
         get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode,
     },
@@ -1000,6 +1002,9 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
                 on_recovery_times: schedule.on_recovery_times,
                 on_recovery_extra_args: schedule.on_recovery_extra_args,
                 ws_error_handler_muted: schedule.ws_error_handler_muted,
+                retry: schedule.retry,
+                summary: schedule.summary,
+                no_flow_overlap: schedule.no_flow_overlap,
             },
         )
         .await;
@@ -2491,21 +2496,23 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
             dedicated_worker,
             None,
         ),
-        JobPayload::Dependencies { hash, dependencies, language, path, dedicated_worker } => (
-            Some(hash.0),
-            Some(path),
-            Some((dependencies, None)),
-            JobKind::Dependencies,
-            None,
-            None,
-            Some(language),
-            None,
-            None,
-            None,
-            dedicated_worker,
-            None,
-        ),
-        JobPayload::FlowDependencies { path, dedicated_worker } => {
+        JobPayload::Dependencies { hash, language, path, dedicated_worker, deployment_message } => {
+            (
+                Some(hash.0),
+                Some(path),
+                Some((deployment_message.unwrap_or_else(String::new), None)),
+                JobKind::Dependencies,
+                None,
+                None,
+                Some(language),
+                None,
+                None,
+                None,
+                dedicated_worker,
+                None,
+            )
+        }
+        JobPayload::FlowDependencies { path, dedicated_worker, deployment_message } => {
             let value_json = fetch_scalar_isolated!(
                 sqlx::query_scalar!(
                     "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
@@ -2523,7 +2530,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
             (
                 None,
                 Some(path),
-                None,
+                Some((deployment_message.unwrap_or_else(String::new), None)),
                 JobKind::FlowDependencies,
                 Some(value.clone()),
                 Some(FlowStatus::new(&value)), // this is a new flow being pushed, flow_status is set to flow_value
@@ -2535,10 +2542,10 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                 None,
             )
         }
-        JobPayload::AppDependencies { path, version } => (
+        JobPayload::AppDependencies { path, version, deployment_message } => (
             Some(version),
             Some(path),
-            None,
+            Some((deployment_message.unwrap_or_else(String::new), None)),
             JobKind::AppDependencies,
             None,
             None,
@@ -2602,6 +2609,63 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
                 value.priority,
             )
         }
+        JobPayload::SingleScriptFlow {
+            path,
+            hash,
+            retry,
+            args,
+            concurrent_limit,
+            concurrency_time_window_s,
+            cache_ttl,
+            priority,
+        } => {
+            let mut input_transforms = HashMap::<String, InputTransform>::new();
+            for (arg_name, arg_value) in args {
+                input_transforms.insert(arg_name, InputTransform::Static { value: arg_value });
+            }
+            let flow_value = FlowValue {
+                modules: vec![FlowModule {
+                    id: "a".to_string(),
+                    value: windmill_common::flows::FlowModuleValue::Script {
+                        input_transforms: input_transforms,
+                        path: path.clone(),
+                        hash: Some(hash),
+                    },
+                    stop_after_if: None,
+                    summary: None,
+                    suspend: None,
+                    mock: None,
+                    retry: Some(retry),
+                    sleep: None,
+                    cache_ttl: None,
+                    timeout: None,
+                    priority: None,
+                    delete_after_use: None,
+                }],
+                same_worker: false,
+                failure_module: None,
+                concurrency_time_window_s: concurrency_time_window_s,
+                concurrent_limit: concurrent_limit,
+                skip_expr: None,
+                cache_ttl: cache_ttl.map(|val| val as u32),
+                early_return: None,
+                priority: priority,
+            };
+            (
+                None,
+                Some(path),
+                None,
+                JobKind::SingleScriptFlow,
+                Some(flow_value.clone()),
+                Some(FlowStatus::new(&flow_value)), // this is a new flow being pushed, flow_status is set to flow_value
+                None,
+                concurrent_limit,
+                concurrency_time_window_s,
+                cache_ttl,
+                None,
+                priority,
+            )
+        }
         JobPayload::Flow { path, dedicated_worker } => {
             let value_json = fetch_scalar_isolated!(
                 sqlx::query_scalar!(
@@ -2907,6 +2971,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
             }
             JobKind::Flow => "jobs.run.flow",
             JobKind::FlowPreview => "jobs.run.flow_preview",
+            JobKind::SingleScriptFlow => "jobs.run.single_script_flow",
             JobKind::Script_Hub => "jobs.run.script_hub",
             JobKind::Dependencies => "jobs.run.dependencies",
             JobKind::Identity => "jobs.run.identity",

backend/windmill-queue/src/schedule.rs

@@ -10,7 +10,9 @@ use crate::push;
 use crate::PushIsolationLevel;
 use crate::QueueTransaction;
 use sqlx::{query_scalar, Postgres, Transaction};
+use std::collections::HashMap;
 use std::str::FromStr;
+use windmill_common::flows::Retry;
 use windmill_common::jobs::JobPayload;
 use windmill_common::schedule::schedule_to_user;
 use windmill_common::DB;
@@ -104,20 +106,50 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             &schedule.script_path,
         )
         .await?;
-        (
-            JobPayload::ScriptHash {
-                hash,
-                path: schedule.script_path,
-                concurrent_limit: concurrent_limit,
-                concurrency_time_window_s: concurrency_time_window_s,
-                cache_ttl: cache_ttl,
-                dedicated_worker,
-                language,
-                priority,
-            },
-            tag,
-            timeout,
-        )
+
+        if schedule.retry.is_some() {
+            let parsed_retry =
+                serde_json::from_value::<Retry>(schedule.retry.unwrap()).map_err(|err| {
+                    error::Error::InternalErr(format!(
+                        "Unable to parse retry information from schedule: {}",
+                        err.to_string(),
+                    ))
+                })?;
+            let mut static_args = HashMap::<String, serde_json::Value>::new();
+            for (arg_name, arg_value) in args.clone() {
+                static_args.insert(arg_name, arg_value);
+            }
+            // if retry is set, we wrap the script into a one step flow with a retry on the module
+            (
+                JobPayload::SingleScriptFlow {
+                    path: schedule.script_path,
+                    hash: hash,
+                    retry: parsed_retry,
+                    args: static_args,
+                    concurrent_limit: concurrent_limit,
+                    concurrency_time_window_s: concurrency_time_window_s,
+                    cache_ttl: cache_ttl,
+                    priority: priority,
+                },
+                Some("flow".to_string()),
+                timeout,
+            )
+        } else {
+            (
+                JobPayload::ScriptHash {
+                    hash,
+                    path: schedule.script_path,
+                    concurrent_limit: concurrent_limit,
+                    concurrency_time_window_s: concurrency_time_window_s,
+                    cache_ttl: cache_ttl,
+                    dedicated_worker,
+                    language,
+                    priority,
+                },
+                tag,
+                timeout,
+            )
+        }
     };
 
     sqlx::query!(

backend/windmill-worker/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/lib.rs"
 
 [features]
 default = []
-enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
+enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
 benchmark = ["windmill-queue/benchmark"]
 flamegraph = []
 
@@ -31,6 +31,7 @@ windmill-parser-py-imports.workspace = true
 windmill-parser-bash.workspace = true
 windmill-parser-sql.workspace = true
 windmill-parser-graphql.workspace = true
+windmill-git-sync.workspace = true
 sqlx.workspace = true
 uuid.workspace = true
 tracing.workspace = true

backend/windmill-worker/loader.py

@@ -28,22 +28,23 @@ class WindmillFinder(MetaPathFinder):
             return ModuleSpec(name, WindmillLoader(name))
         elif l > 2:
             script_path = "/".join(splitted)
-            import requests
+            import urllib.parse
+            import urllib.request
 
+            headers = {"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
             url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
 
-            r = requests.get(
-                url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
-            )
-
-            if r.status_code == 200:
-                folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
-                fullpath = folder + "/" + splitted[-1] + ".py"
-                os.makedirs(folder, exist_ok=True)
-                with open(fullpath, "w+") as f:
-                    f.write(r.text)
-                return ModuleSpec(name, SourceFileLoader(name, fullpath))
-            else:
+            req = urllib.request.Request(url, None, headers)
+            try:
+                with urllib.request.urlopen(req) as response:
+                        r = response.read().decode("utf-8")
+                        folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1])
+                        fullpath = folder + "/" + splitted[-1] + ".py"
+                        os.makedirs(folder, exist_ok=True)
+                        with open(fullpath, "w+") as f:
+                            f.write(r)
+                        return ModuleSpec(name, SourceFileLoader(name, fullpath))
+            except:
                 # raise ImportError(f"Script {script_path} not found")
                 return ModuleSpec(name, WindmillLoader(name))
 

backend/windmill-worker/src/bun_executor.rs

@@ -16,7 +16,7 @@ use crate::{
         start_child_process, write_file, write_file_binary,
     },
     AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV,
-    NSJAIL_PATH, PATH_ENV, TZ_ENV,
+    NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
 
 use tokio::{
@@ -84,6 +84,38 @@ pub async fn gen_lockfile(
     )
     .await?;
 
+    // if custom NPM registry is being used, write bunfig.toml at the root of the job dir
+    if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
+        let (url, token_opt) = if s.contains(":_authToken=") {
+            let split_url = s.split(":_authToken=").collect::<Vec<&str>>();
+            let url = split_url
+                .get(0)
+                .map(|u| u.to_string())
+                .unwrap_or("".to_string());
+            let token = split_url
+                .get(1)
+                .map(|t| t.to_string())
+                .unwrap_or("".to_string());
+            (url, Some(token))
+        } else {
+            (s.to_owned(), None)
+        };
+        let registry_toml_string = if let Some(token) = token_opt {
+            format!("{{ url = \"{url}\", token = \"{token}\" }}")
+        } else {
+            format!("\"{url}\"")
+        };
+        let bunfig_toml = format!(
+            r#"
+[install]
+registry = {}
+"#,
+            registry_toml_string
+        );
+        tracing::debug!("Writing following bunfig.toml: {bunfig_toml}");
+        let _ = write_file(&job_dir, "bunfig.toml", &bunfig_toml).await?;
+    }
+
     let common_bun_proc_envs: HashMap<String, String> =
         get_common_bun_proc_envs(&base_internal_url).await;
 
@@ -304,26 +336,26 @@ pub async fn handle_bun_job(
         let trusted_deps = get_trusted_deps(inner_content);
         let empty_trusted_deps = trusted_deps.len() == 0;
         let has_custom_config_registry = common_bun_proc_envs.contains_key("NPM_CONFIG_REGISTRY");
-        if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
-            logs.push_str("\n\n--- BUN INSTALL ---\n");
-            set_logs(&logs, &job.id, &db).await;
-            let _ = gen_lockfile(
-                logs,
-                mem_peak,
-                canceled_by,
-                &job.id,
-                &job.workspace_id,
-                db,
-                &client.get_token().await,
-                &job.script_path(),
-                job_dir,
-                base_internal_url,
-                worker_name,
-                false,
-                trusted_deps,
-            )
-            .await?;
-        }
+        // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
+        logs.push_str("\n\n--- BUN INSTALL ---\n");
+        set_logs(&logs, &job.id, &db).await;
+        let _ = gen_lockfile(
+            logs,
+            mem_peak,
+            canceled_by,
+            &job.id,
+            &job.workspace_id,
+            db,
+            &client.get_token().await,
+            &job.script_path(),
+            job_dir,
+            base_internal_url,
+            worker_name,
+            false,
+            trusted_deps,
+        )
+        .await?;
+        // }
 
         if empty_trusted_deps && !has_custom_config_registry {
             let node_modules_path = format!("{}/node_modules", job_dir);
@@ -680,9 +712,9 @@ for await (const chunk of Bun.stdin.stream()) {{
         try {{
             let {{ {spread} }} = JSON.parse(line) 
             let res: any = await main(...[ {spread} ]);
-            stdout.write("wm_res:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n');
+            stdout.write("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n');
         }} catch (e) {{
-            stdout.write("wm_res:" + JSON.stringify({{ error: {{ message: e.message, name: e.name, stack: e.stack, line: line }}}}) + '\n');
+            stdout.write("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n');
         }}
         stdout.flush();
     }}

backend/windmill-worker/src/common.rs

@@ -3,12 +3,13 @@ use itertools::Itertools;
 use nix::sys::signal::{self, Signal};
 use nix::unistd::Pid;
 use regex::Regex;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::value::RawValue;
 use serde_json::{json, Value};
 use sqlx::{Pool, Postgres};
 use tokio::process::Command;
 use tokio::{fs::File, io::AsyncReadExt};
+use windmill_common::s3_helpers::{get_etag_or_empty, LargeFileStorage, S3Object, S3Resource};
 use windmill_common::worker::{CLOUD_HOSTED, WORKER_CONFIG};
 use windmill_common::{
     error::{self, Error},
@@ -31,7 +32,7 @@ use std::{
 
 use tracing::{trace_span, Instrument};
 use uuid::Uuid;
-use windmill_common::variables;
+use windmill_common::{job_metrics, variables, DB};
 
 use tokio::{
     io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
@@ -407,6 +408,7 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
         }
         -2
     } else {
+        // rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
         -3
     }
 }
@@ -426,7 +428,7 @@ pub async fn handle_child(
     mut child: Child,
     nsjail: bool,
     worker_name: &str,
-    _w_id: &str,
+    w_id: &str,
     child_name: &str,
     custom_timeout: Option<i32>,
     sigterm: bool,
@@ -464,6 +466,8 @@ pub async fn handle_child(
         interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
 
         let mut i = 0;
+        let mut memory_metric_id: Result<String, Error> =
+            Err(Error::NotFound("not yet initialized".to_string()));
 
         loop {
             tokio::select!(
@@ -484,7 +488,29 @@ pub async fn handle_child(
                     if current_mem > *mem_peak {
                         *mem_peak = current_mem
                     }
-                    tracing::info!("{worker_name}/{job_id} in {_w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
+                    tracing::info!("{worker_name}/{job_id} in {w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
+
+                    #[cfg(feature = "enterprise")]
+                    {
+                        // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
+                        if i == 2 {
+                            memory_metric_id = job_metrics::register_metric_for_job(
+                                &db,
+                                w_id.to_string(),
+                                job_id,
+                                "memory_kb".to_string(),
+                                job_metrics::MetricKind::TimeseriesInt,
+                                Some("Job Memory Footprint (kB)".to_string()),
+                            )
+                            .await;
+                        }
+                        if let Ok(ref metric_id) = memory_metric_id {
+                            if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
+                                tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
+                            }
+                        }
+                    }
+
                     let (canceled, canceled_by, canceled_reason) = sqlx::query_as::<_, (bool, Option<String>, Option<String>)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason")
                         .bind(*mem_peak)
                         .bind(job_id)
@@ -515,7 +541,7 @@ pub async fn handle_child(
     }
 
     let (timeout_duration, timeout_warn_msg) =
-        resolve_job_timeout(&db, _w_id, job_id, custom_timeout).await;
+        resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
     if let Some(msg) = timeout_warn_msg {
         logs.push_str(msg.as_str());
         append_logs(job_id, msg.as_str(), db).await;
@@ -585,7 +611,11 @@ pub async fn handle_child(
         };
 
         /* log_remaining is zero when output limit was reached */
-        let mut log_remaining = max_log_size.saturating_sub(logs.chars().count());
+        let mut log_remaining =  if *CLOUD_HOSTED {
+            max_log_size.saturating_sub(logs.chars().count())
+        } else {
+            usize::MAX
+        };
         let mut result = io::Result::Ok(());
         let mut output = output.take_until(rx2.recv()).boxed();
         /* `do_write` resolves the task, but does not contain the Result.
@@ -615,7 +645,7 @@ pub async fn handle_child(
                         if line.is_empty() {
                             continue;
                         }
-                        append_with_limit(&mut joined, &line, &mut log_remaining);
+                        append_with_limit(&mut joined, &line, &mut log_remaining, child_name == "powershell run" || child_name == "bash run");
                         if log_remaining == 0 {
                             tracing::info!(%job_id, "Too many logs lines for job {job_id}");
                             let _ = set_too_many_logs.send(true);
@@ -802,11 +832,24 @@ pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
 
 // as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
 // so this pushes \n to thd destination string in each call
-fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
-    if *limit > 0 {
+fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize, remove_x00: bool) {
+    let src_str;
+    let src = if remove_x00 {
+        src_str = src.replace('\u{00}', "");
+        src_str.as_str()
+    } else {
+        src
+    };
+    if !*CLOUD_HOSTED {
         dst.push('\n');
+        dst.push_str(&src);
+        return;
+    } else {
+        if *limit > 0 {
+            dst.push('\n');
+        }
+        *limit -= 1;
     }
-    *limit -= 1;
 
     let src_len = src.chars().count();
     if src_len <= *limit {
@@ -824,13 +867,26 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
     }
 }
 
-pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> String {
+pub async fn hash_args(
+    db: &DB,
+    client: &AuthedClient,
+    workspace_id: &str,
+    job_id: &Uuid,
+    v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
+) -> String {
     if let Some(vs) = v {
         let mut dh = DefaultHasher::new();
         let hm = &vs.0;
         for k in hm.keys().sorted() {
             k.hash(&mut dh);
-            hm.get(k).unwrap().get().hash(&mut dh);
+            let arg_value = hm.get(k).unwrap();
+            let arg_additions =
+                arg_value_hash_additions(db, client, workspace_id, job_id, hm.get(k).unwrap())
+                    .await;
+            arg_value.get().hash(&mut dh);
+            for (_, arg_addition) in arg_additions {
+                arg_addition.hash(&mut dh);
+            }
         }
         hex::encode(dh.finish().to_be_bytes())
     } else {
@@ -838,21 +894,152 @@ pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>)
     }
 }
 
-#[derive(Serialize)]
-struct StoreCachedResource<'a> {
-    expire: i64,
-    value: &'a RawValue,
+async fn get_workspace_s3_resource_path(
+    db: &DB,
+    client: &AuthedClient,
+    workspace_id: &str,
+    job_id: &Uuid,
+) -> Option<S3Resource> {
+    let raw_lfs_opt = sqlx::query_scalar!(
+        "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
+        workspace_id
+    )
+    .fetch_one(db)
+    .await
+    .ok()
+    .flatten()
+    .map(|val| serde_json::from_value::<LargeFileStorage>(val).ok())
+    .flatten();
+
+    if let Some(LargeFileStorage::S3Storage(s3_storage)) = raw_lfs_opt {
+        let resource_path = s3_storage.s3_resource_path.trim_start_matches("$res:");
+        client
+            .get_resource_value_interpolated::<S3Resource>(&resource_path, Some(job_id.to_string()))
+            .await
+            .ok()
+    } else {
+        return None;
+    }
 }
 
-pub async fn save_in_cache<'a>(
+async fn arg_value_hash_additions(
+    db: &DB,
+    client: &AuthedClient,
+    workspace_id: &str,
+    job_id: &Uuid,
+    raw_value: &Box<RawValue>,
+) -> HashMap<String, String> {
+    let mut result: HashMap<String, String> = HashMap::new();
+    let parsed_s3_values: &mut Vec<S3Object> = &mut Vec::new();
+    extract_all_s3_object_from_raw_value(raw_value, parsed_s3_values);
+    if parsed_s3_values.is_empty() {
+        // no s3 object, nothing to return
+        return result;
+    }
+
+    let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id, job_id).await;
+    if let Some(s3_resource) = s3_resource_opt {
+        for s3_object in parsed_s3_values {
+            let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await;
+            tracing::warn!("Enriching s3 arg value with etag: {:?}", etag);
+            result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache?
+        }
+    }
+
+    return result;
+}
+
+fn extract_all_s3_object_from_raw_value(raw_value: &Box<RawValue>, result: &mut Vec<S3Object>) {
+    let parsed_value = serde_json::from_str::<S3Object>(raw_value.get());
+    if let Ok(parsed_value) = parsed_value {
+        result.push(parsed_value);
+    } else {
+        let parsed_value = serde_json::from_str::<HashMap<String, Box<RawValue>>>(raw_value.get());
+        if let Ok(parsed_value) = parsed_value {
+            for (_, v) in parsed_value {
+                extract_all_s3_object_from_raw_value(&v, result);
+            }
+        } else {
+            let parsed_value = serde_json::from_str::<Vec<Box<RawValue>>>(raw_value.get());
+            if let Ok(parsed_value) = parsed_value {
+                for v in parsed_value {
+                    extract_all_s3_object_from_raw_value(&v, result);
+                }
+            }
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+struct CachedResource {
+    expire: i64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    s3_etags: Option<HashMap<String, String>>,
+    value: Box<RawValue>,
+}
+
+pub async fn get_cached_resource_value_if_valid(
+    db: &DB,
+    client: &AuthedClient,
+    job_id: &Uuid,
+    workspace_id: &str,
+    cached_res_path: &str,
+) -> Option<Box<RawValue>> {
+    let resource_opt = client
+        .get_resource_value::<CachedResource>(cached_res_path)
+        .await
+        .ok();
+    if let Some(cached_resource) = resource_opt {
+        if cached_resource.expire <= chrono::Utc::now().timestamp() {
+            // cache expired
+            return None;
+        }
+        let s3_etags = cached_resource.s3_etags.unwrap_or_default();
+        let s3_resource_opt: Option<S3Resource> = if s3_etags.is_empty() {
+            None
+        } else {
+            get_workspace_s3_resource_path(db, &client, workspace_id, job_id).await
+        };
+        if !s3_etags.is_empty() && s3_resource_opt.is_none() {
+            tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated");
+            return None;
+        }
+        for (s3_file_key, s3_file_etag) in s3_etags {
+            if let Some(s3_resource) = s3_resource_opt.clone() {
+                let etag =
+                    get_etag_or_empty(&s3_resource, S3Object { s3: s3_file_key.clone() }).await;
+                if etag.is_none() || etag.clone().unwrap() != s3_file_etag {
+                    tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag);
+                    return None;
+                }
+            }
+        }
+        return Some(cached_resource.value);
+    }
+    return None;
+}
+
+pub async fn save_in_cache(
     db: &Pool<Postgres>,
+    client: &AuthedClient,
     job: &QueuedJob,
     cached_path: String,
-    r: &'a RawValue,
+    r: &Box<RawValue>,
 ) {
     let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64;
 
-    let store_cache_resource = StoreCachedResource { expire, value: r };
+    let s3_etags =
+        arg_value_hash_additions(db, client, job.workspace_id.as_str(), &job.id, r).await;
+
+    let store_cache_resource = CachedResource {
+        expire,
+        s3_etags: if s3_etags.is_empty() {
+            None
+        } else {
+            Some(s3_etags)
+        },
+        value: r.clone(),
+    };
     let raw_json = sqlx::types::Json(store_cache_resource);
 
     if let Err(e) = sqlx::query!(
@@ -862,7 +1049,7 @@ pub async fn save_in_cache<'a>(
     DO UPDATE SET value = $3",
         job.workspace_id,
         cached_path,
-        raw_json as sqlx::types::Json<StoreCachedResource>,
+        raw_json as sqlx::types::Json<CachedResource>,
         "cache"
     )
     .execute(db)

backend/windmill-worker/src/dedicated_worker.rs

@@ -104,8 +104,7 @@ pub async fn handle_dedicated_process(
             .wait()
             .await
             .expect("child process encountered an error");
-
-        println!("child status was: {}", status);
+        tracing::info!("child status was: {}", status);
     });
 
     let mut jobs = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS);
@@ -144,10 +143,16 @@ pub async fn handle_dedicated_process(
                         continue;
                     }
                     tracing::debug!("processed job: {line}");
-                    if line.starts_with("wm_res:") {
+                    if line.starts_with("wm_res[") {
                         let job: Arc<QueuedJob> = jobs.pop_front().expect("pop");
-                        match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line.replace("wm_res:", "")) {
-                            Ok(result) => job_completed_tx.send(JobCompleted { job , result, logs: logs, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap(),
+                        match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) {
+                            Ok(result) => {
+                                if line.starts_with("wm_res[success]:") {
+                                    job_completed_tx.send(JobCompleted { job , result, logs: logs, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap()
+                                } else {
+                                    job_completed_tx.send(JobCompleted { job , result, logs: logs, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string() }).await.unwrap()
+                                }
+                            },
                             Err(e) => {
                                 tracing::error!("Could not deserialize job result `{line}`: {e:?}");
                                 job_completed_tx.send(JobCompleted { job , result: to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")})), logs: "".to_string(), mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string() }).await.unwrap();

backend/windmill-worker/src/deno_executor.rs

@@ -463,9 +463,9 @@ for await (const chunk of Deno.stdin.readable) {{
         try {{
             let {{ {spread} }} = JSON.parse(line) 
             let res: any = await main(...[ {spread} ]);
-            console.log("wm_res:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n');
+            console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n');
         }} catch (e) {{
-            console.log("wm_res:" + JSON.stringify({{ error: {{ message: e.message, name: e.name, stack: e.stack, line: line }}}}) + '\n');
+            console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n');
         }}
     }}
     if (exit) {{

backend/windmill-worker/src/pg_executor.rs

@@ -322,7 +322,13 @@ fn convert_val(value: &Value, arg_t: &String) -> windmill_common::error::Result<
         Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => {
             Ok(PgType::I16(n.as_i64().unwrap() as i16))
         }
-        Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "serial") => {
+        Value::Number(n)
+            if n.is_i64()
+                && (arg_t == "int"
+                    || arg_t == "integer"
+                    || arg_t == "int4"
+                    || arg_t == "serial") =>
+        {
             Ok(PgType::I32(n.as_i64().unwrap() as i32))
         }
         Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
@@ -429,7 +435,6 @@ pub fn pg_cell_to_json_value(
         Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| {
             Ok(JSONValue::String(a.0))
         })?,
-
         // array types
         Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
         Type::INT2_ARRAY => get_array(row, column, column_i, |a: i16| {

backend/windmill-worker/src/python_executor.rs

@@ -27,8 +27,7 @@ lazy_static::lazy_static! {
 
     static ref FLOCK_PATH: String =
     std::env::var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string());
-
-
+    static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap();
 
     static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
     static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
@@ -84,14 +83,24 @@ pub async fn pip_compile(
         WORKER_CONFIG.read().await.pip_local_dependencies.as_ref()
     {
         let deps = pip_local_dependencies.clone();
+        let compiled_deps = deps.iter().map(|dep| {
+            let compiled_dep = Regex::new(dep);
+            match compiled_dep {
+                Ok(compiled_dep) => Some(compiled_dep),
+                Err(e) => {
+                    tracing::warn!("regex compilation failed for Python local dependency: '{}' - it will be ignored", e);
+                    return None;
+                }
+            }
+        }).filter(|dep_maybe| dep_maybe.is_some()).map(|dep| dep.unwrap()).collect::<Vec<Regex>>();
         requirements
             .lines()
             .filter(|s| {
-                if !deps.contains(&s.to_string()) {
-                    return true;
-                } else {
+                if compiled_deps.iter().any(|dep| dep.is_match(s)) {
                     logs.push_str(&format!("\nignoring local dependency: {}", s));
                     return false;
+                } else {
+                    return true;
                 }
             })
             .join("\n")
@@ -262,7 +271,7 @@ try:
     res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
     with open("result.json", 'w') as f:
         f.write(res_json)
-except Exception as e:
+except BaseException as e:
     exc_type, exc_value, exc_traceback = sys.exc_info()
     tb = traceback.format_tb(exc_traceback)
     with open("result.json", 'w') as f:
@@ -567,7 +576,7 @@ async fn handle_python_deps(
     };
 
     if requirements.len() > 0 {
-        additional_python_paths = handle_python_reqs(
+        let mut venv_path = handle_python_reqs(
             requirements
                 .split("\n")
                 .filter(|x| !x.starts_with("--"))
@@ -583,6 +592,7 @@ async fn handle_python_deps(
             worker_dir,
         )
         .await?;
+        additional_python_paths.append(&mut venv_path);
     }
     Ok(additional_python_paths)
 }
@@ -695,6 +705,8 @@ pub async fn handle_python_reqs(
                 .stderr(Stdio::piped());
             start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
         } else {
+            let fssafe_req = NON_ALPHANUM_CHAR.replace_all(req, "_").to_string();
+            let req = format!("'{}'", req);
             let mut command_args = vec![
                 PYTHON_PATH.as_str(),
                 "-m",
@@ -740,7 +752,7 @@ pub async fn handle_python_reqs(
                 .envs(envs)
                 .args([
                     "-x",
-                    &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, req),
+                    &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
                     "--command",
                     &command_args.join(" "),
                 ])
@@ -924,12 +936,12 @@ for line in sys.stdin:
                 if type(v).__name__ == 'bytes':
                     res[k] = to_b_64(v)
         res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
-        sys.stdout.write("wm_res:" + res_json + "\n")
-    except Exception as e:
+        sys.stdout.write("wm_res[success]:" + res_json + "\n")
+    except BaseException as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         tb = traceback.format_tb(exc_traceback)
-        err_json = json.dumps({{ "error": {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:])  }} }}, separators=(',', ':'), default=str).replace('\n', '')
-        sys.stdout.write("wm_res:" + err_json + "\n")
+        err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:])  }}, separators=(',', ':'), default=str).replace('\n', '')
+        sys.stdout.write("wm_res[error]:" + err_json + "\n")
     sys.stdout.flush()
 "#,
         );

backend/windmill-worker/src/worker.rs

@@ -25,6 +25,8 @@ use std::{
     },
     time::Duration,
 };
+use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
+use windmill_parser_py_imports::parse_relative_imports;
 
 use uuid::Uuid;
 #[cfg(feature = "enterprise")]
@@ -33,9 +35,9 @@ use windmill_common::{
     error::{self, to_anyhow, Error},
     flows::{FlowModule, FlowModuleValue, FlowValue},
     get_latest_deployed_hash_for_path,
-    jobs::{JobKind, QueuedJob},
+    jobs::{JobKind, JobPayload, QueuedJob},
     scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
-    users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
+    users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
     utils::{rd_string, StripPath},
     worker::{
         to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG, WORKER_GROUP,
@@ -76,7 +78,10 @@ use windmill_queue::{add_completed_job, add_completed_job_error};
 use crate::{
     bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE},
     bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job},
-    common::{build_args_map, hash_args, read_result, save_in_cache, write_file},
+    common::{
+        build_args_map, get_cached_resource_value_if_valid, hash_args, read_result, save_in_cache,
+        write_file,
+    },
     deno_executor::{generate_deno_lock, handle_deno_job},
     go_executor::{handle_go_job, install_go_dependencies},
     graphql_executor::do_graphql,
@@ -146,7 +151,8 @@ pub async fn create_token_for_owner(
             .await?
             .unwrap_or(false)
             || email == SUPERADMIN_SECRET_EMAIL
-            || email == SUPERADMIN_NOTIFICATION_EMAIL;
+            || email == SUPERADMIN_NOTIFICATION_EMAIL
+            || owner == SUPERADMIN_SYNC_EMAIL;
 
     sqlx::query_scalar!(
         "INSERT INTO token
@@ -469,6 +475,7 @@ async fn handle_receive_completed_job<
     worker_name: &str,
     worker_save_completed_job_duration: Option<Arc<prometheus::Histogram>>,
     worker_flow_transition_duration: Option<Arc<prometheus::Histogram>>,
+    job_completed_tx: Sender<SendResult>,
 ) {
     let token = jc.token.clone();
     let workspace = jc.job.workspace_id.clone();
@@ -491,6 +498,7 @@ async fn handle_receive_completed_job<
         worker_name,
         worker_save_completed_job_duration,
         worker_flow_transition_duration,
+        job_completed_tx.clone(),
     )
     .await
     {
@@ -506,6 +514,7 @@ async fn handle_receive_completed_job<
             &worker_dir,
             rsmq.clone(),
             worker_name,
+            job_completed_tx,
         )
         .await;
     }
@@ -513,7 +522,7 @@ async fn handle_receive_completed_job<
 
 #[derive(Clone)]
 pub struct JobCompletedSender(
-    Sender<JobCompleted>,
+    Sender<SendResult>,
     Option<Arc<GenericGauge<AtomicI64>>>,
     Option<Arc<prometheus::Histogram>>,
 );
@@ -522,12 +531,12 @@ impl JobCompletedSender {
     pub async fn send(
         &self,
         jc: JobCompleted,
-    ) -> Result<(), tokio::sync::mpsc::error::SendError<JobCompleted>> {
+    ) -> Result<(), tokio::sync::mpsc::error::SendError<SendResult>> {
         if let Some(wj) = self.1.as_ref() {
             wj.inc()
         }
         let timer = self.2.as_ref().map(|x| x.start_timer());
-        let r = self.0.send(jc).await;
+        let r = self.0.send(SendResult::JobCompleted(jc)).await;
         timer.map(|x| x.stop_and_record());
         r
     }
@@ -873,7 +882,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<Uuid>(5);
 
-    let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<JobCompleted>(3);
+    let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<SendResult>(3);
 
     let job_completed_tx = JobCompletedSender(
         job_completed_tx,
@@ -961,119 +970,167 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     let worker_name2 = worker_name.clone();
     let killpill_tx2 = killpill_tx.clone();
+    let job_completed_sender = job_completed_tx.0.clone();
     let send_result = tokio::spawn(async move {
-        while let Some(jc) = job_completed_rx.recv().await {
-            if let Some(wj) = worker_job_completed_channel_queue2.as_ref() {
-                wj.dec();
-            }
-            let base_internal_url2 = base_internal_url2.clone();
-            let worker_dir2 = worker_dir2.clone();
-            let db2 = db2.clone();
-            let same_worker_tx2 = same_worker_tx2.clone();
-            let rsmq2 = rsmq2.clone();
-            let worker_name = worker_name2.clone();
-            if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
-                thread_count.fetch_add(1, Ordering::SeqCst);
-                let thread_count = thread_count.clone();
-
-                loop {
-                    if thread_count.load(Ordering::Relaxed) < 4 {
-                        break;
+        while let Some(sr) = job_completed_rx.recv().await {
+            match sr {
+                SendResult::JobCompleted(jc) => {
+                    if let Some(wj) = worker_job_completed_channel_queue2.as_ref() {
+                        wj.dec();
                     }
-                    tokio::time::sleep(Duration::from_millis(3)).await;
-                }
+                    let base_internal_url2 = base_internal_url2.clone();
+                    let worker_dir2 = worker_dir2.clone();
+                    let db2 = db2.clone();
+                    let same_worker_tx2 = same_worker_tx2.clone();
+                    let rsmq2 = rsmq2.clone();
+                    let worker_name = worker_name2.clone();
+                    if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
+                        thread_count.fetch_add(1, Ordering::SeqCst);
+                        let thread_count = thread_count.clone();
 
-                #[cfg(feature = "benchmark")]
-                let send_duration = send_duration2.clone();
-                #[cfg(feature = "benchmark")]
-                let process_duration = process_duration.clone();
-                #[cfg(feature = "benchmark")]
-                let completed_jobs = completed_jobs.clone();
-                #[cfg(feature = "benchmark")]
-                let main_duration = main_duration2.clone();
-
-                let worker_save_completed_job_duration2 =
-                    worker_save_completed_job_duration.clone();
-                let worker_flow_transition_duration2 = worker_flow_transition_duration.clone();
-                let killpill_tx = killpill_tx2.clone();
-                tokio::spawn(async move {
-                    #[cfg(feature = "benchmark")]
-                    let process_start = Instant::now();
-
-                    let is_dependency_job = matches!(
-                        jc.job.job_kind,
-                        JobKind::Dependencies | JobKind::FlowDependencies
-                    );
-                    handle_receive_completed_job(
-                        jc,
-                        base_internal_url2,
-                        db2.clone(),
-                        worker_dir2,
-                        same_worker_tx2,
-                        rsmq2,
-                        &worker_name,
-                        worker_save_completed_job_duration2.clone(),
-                        worker_flow_transition_duration2.clone(),
-                    )
-                    .await;
-                    #[cfg(feature = "benchmark")]
-                    {
-                        let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
-                        if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
-                            let duration_s = start.elapsed().as_secs_f64();
-                            let jobs_per_sec = n as f64 / duration_s;
-                            tracing::info!(
-                                "completed {} jobs in {}s, {} jobs/s",
-                                n + 1,
-                                duration_s,
-                                jobs_per_sec
-                            );
-
-                            tracing::info!(
-                                "main loop without send {}s",
-                                main_duration.load(Ordering::SeqCst) as f64 / 1000.0
-                            );
-
-                            tracing::info!(
-                                "send job completed / send dedicated job duration {}s",
-                                send_duration.load(Ordering::SeqCst) as f64 / 1000.0
-                            );
-
-                            tracing::info!(
-                                "job completed process duration {}s",
-                                process_duration.load(Ordering::SeqCst) as f64 / 1000.0
-                            );
+                        loop {
+                            if thread_count.load(Ordering::Relaxed) < 4 {
+                                break;
+                            }
+                            tokio::time::sleep(Duration::from_millis(3)).await;
                         }
 
-                        process_duration.fetch_add(
-                            process_start.elapsed().as_millis() as usize,
-                            Ordering::SeqCst,
-                        );
-                    }
+                        #[cfg(feature = "benchmark")]
+                        let send_duration = send_duration2.clone();
+                        #[cfg(feature = "benchmark")]
+                        let process_duration = process_duration.clone();
+                        #[cfg(feature = "benchmark")]
+                        let completed_jobs = completed_jobs.clone();
+                        #[cfg(feature = "benchmark")]
+                        let main_duration = main_duration2.clone();
 
-                    thread_count.fetch_sub(1, Ordering::SeqCst);
-                    if is_dependency_job && is_dedicated_worker {
-                        tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
-                        sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
+                        let worker_save_completed_job_duration2 =
+                            worker_save_completed_job_duration.clone();
+                        let worker_flow_transition_duration2 =
+                            worker_flow_transition_duration.clone();
+                        let killpill_tx = killpill_tx2.clone();
+                        let job_completed_sender = job_completed_sender.clone();
+                        tokio::spawn(async move {
+                            #[cfg(feature = "benchmark")]
+                            let process_start = Instant::now();
+
+                            let is_dependency_job = matches!(
+                                jc.job.job_kind,
+                                JobKind::Dependencies | JobKind::FlowDependencies
+                            );
+                            handle_receive_completed_job(
+                                jc,
+                                base_internal_url2,
+                                db2.clone(),
+                                worker_dir2,
+                                same_worker_tx2,
+                                rsmq2,
+                                &worker_name,
+                                worker_save_completed_job_duration2.clone(),
+                                worker_flow_transition_duration2.clone(),
+                                job_completed_sender.clone(),
+                            )
+                            .await;
+                            #[cfg(feature = "benchmark")]
+                            {
+                                let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
+                                if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
+                                    let duration_s = start.elapsed().as_secs_f64();
+                                    let jobs_per_sec = n as f64 / duration_s;
+                                    tracing::info!(
+                                        "completed {} jobs in {}s, {} jobs/s",
+                                        n + 1,
+                                        duration_s,
+                                        jobs_per_sec
+                                    );
+
+                                    tracing::info!(
+                                        "main loop without send {}s",
+                                        main_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                                    );
+
+                                    tracing::info!(
+                                        "send job completed / send dedicated job duration {}s",
+                                        send_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                                    );
+
+                                    tracing::info!(
+                                        "job completed process duration {}s",
+                                        process_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                                    );
+                                }
+
+                                process_duration.fetch_add(
+                                    process_start.elapsed().as_millis() as usize,
+                                    Ordering::SeqCst,
+                                );
+                            }
+
+                            thread_count.fetch_sub(1, Ordering::SeqCst);
+                            if is_dependency_job && is_dedicated_worker {
+                                tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
+                                sqlx::query!("UPDATE config SET config = config WHERE name = $1", format!("worker__{}", *WORKER_GROUP))
                             .execute(&db2)
                             .await
                             .expect("update config to trigger restart of all dedicated workers at that config");
-                        killpill_tx.send(()).unwrap_or_default();
+                                killpill_tx.send(()).unwrap_or_default();
+                            }
+                        });
+                    } else {
+                        handle_receive_completed_job(
+                            jc,
+                            base_internal_url2,
+                            db2,
+                            worker_dir2,
+                            same_worker_tx2,
+                            rsmq2,
+                            &worker_name,
+                            worker_save_completed_job_duration2.clone(),
+                            worker_flow_transition_duration2.clone(),
+                            job_completed_sender.clone(),
+                        )
+                        .await;
                     }
-                });
-            } else {
-                handle_receive_completed_job(
-                    jc,
-                    base_internal_url2,
-                    db2,
-                    worker_dir2,
-                    same_worker_tx2,
-                    rsmq2,
-                    &worker_name,
-                    worker_save_completed_job_duration2.clone(),
-                    worker_flow_transition_duration2.clone(),
-                )
-                .await;
+                }
+                SendResult::UpdateFlow {
+                    flow,
+                    w_id,
+                    success,
+                    result,
+                    worker_dir,
+                    stop_early_override,
+                    token,
+                } => {
+                    // let r;
+                    if let Err(e) = update_flow_status_after_job_completion(
+                        &db2,
+                        &AuthedClient {
+                            base_internal_url: base_internal_url2.to_string(),
+                            workspace: w_id.clone(),
+                            token: token.clone(),
+                            force_client: None,
+                        },
+                        flow,
+                        &Uuid::nil(),
+                        &w_id,
+                        success,
+                        &result,
+                        true,
+                        same_worker_tx2.clone(),
+                        &worker_dir,
+                        stop_early_override,
+                        rsmq2.clone(),
+                        &worker_name2,
+                        job_completed_sender.clone(),
+                    )
+                    .await
+                    {
+                        tracing::error!("Error updating flow status after job completion: {e}");
+                    }
+                }
+                SendResult::Kill => {
+                    break;
+                }
             }
         }
 
@@ -1330,6 +1387,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         }
                     }
                     println!("received killpill for worker {}", i_worker);
+                    job_completed_tx.0.send(SendResult::Kill).await.unwrap();
                     break
                 },
                 _ = copy_to_bucket_rx.recv() => {
@@ -1597,6 +1655,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                             &worker_dir,
                             rsmq.clone(),
                             &worker_name,
+                            (&job_completed_tx.0).clone(),
                         )
                         .await;
                     };
@@ -2080,11 +2139,12 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
     worker_name: &str,
     worker_save_completed_job_duration: Option<Arc<prometheus::Histogram>>,
     worker_flow_transition_duration: Option<Arc<prometheus::Histogram>>,
+    job_completed_tx: Sender<SendResult>,
 ) -> windmill_common::error::Result<()> {
     if success {
         // println!("bef completed job{:?}",  SystemTime::now());
         if let Some(cached_path) = cached_res_path {
-            save_in_cache(db, &job, cached_path.to_string(), &result).await;
+            save_in_cache(db, client, &job, cached_path.to_string(), &result).await;
         }
 
         let timer = worker_save_completed_job_duration
@@ -2123,6 +2183,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
                     None,
                     rsmq.clone(),
                     worker_name,
+                    job_completed_tx,
                 )
                 .await?;
                 timer.map(|x| x.stop_and_record());
@@ -2158,6 +2219,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
                     None,
                     rsmq,
                     worker_name,
+                    job_completed_tx,
                 )
                 .await?;
             }
@@ -2213,6 +2275,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
     worker_dir: &str,
     rsmq: Option<R>,
     worker_name: &str,
+    job_completed_tx: Sender<SendResult>,
 ) {
     let err = match err {
         Error::JsonErr(err) => err,
@@ -2260,6 +2323,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
             None,
             rsmq.clone(),
             worker_name,
+            job_completed_tx.clone(),
         )
         .await;
 
@@ -2302,6 +2366,34 @@ fn extract_error_value(log_lines: &str, i: i32) -> Box<RawValue> {
     );
 }
 
+pub enum SendResult {
+    JobCompleted(JobCompleted),
+    UpdateFlow {
+        flow: Uuid,
+        w_id: String,
+        success: bool,
+        result: Box<RawValue>,
+        worker_dir: String,
+        stop_early_override: Option<bool>,
+        token: String,
+    },
+    Kill,
+}
+
+// db: &DB,
+// client: &AuthedClient,
+// flow: uuid::Uuid,
+// job_id_for_status: &Uuid,
+// w_id: &str,
+// success: bool,
+// result: &'a RawValue,
+// unrecoverable: bool,
+// same_worker_tx: Sender<Uuid>,
+// worker_dir: &str,
+// stop_early_override: Option<bool>,
+// rsmq: Option<R>,
+// worker_name: &str,
+
 #[derive(Debug, Clone)]
 pub struct JobCompleted {
     pub job: Arc<QueuedJob>,
@@ -2332,12 +2424,6 @@ async fn do_nativets(
     Ok((result.0, [logs, result.1].join("\n\n")))
 }
 
-#[derive(Deserialize)]
-struct CachedResource {
-    expire: i64,
-    value: Box<RawValue>,
-}
-
 #[derive(Deserialize, Serialize, Default)]
 pub struct PreviousResult<'a> {
     #[serde(borrow)]
@@ -2399,7 +2485,14 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         } else {
             "none".to_string()
         };
-        let args_hash = hash_args(&job.args);
+        let args_hash = hash_args(
+            db,
+            &client.get_authed().await,
+            &job.workspace_id,
+            &job.id,
+            &job.args,
+        )
+        .await;
         if job.is_flow_step {
             let flow_path = sqlx::query_scalar!(
                 "SELECT script_path FROM queue WHERE id = $1",
@@ -2424,37 +2517,36 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
 
     if let Some(cached_res_path) = cached_res_path.clone() {
         let authed_client = client.get_authed().await;
-        let resource = authed_client
-            .get_resource_value::<CachedResource>(&cached_res_path)
-            .await;
 
-        if let Ok(resource) = resource {
-            let expire = resource.expire;
-            if expire > chrono::Utc::now().timestamp() {
-                let result = resource.value;
-                let logs =
-                    "Job skipped because args & path found in cache and not expired".to_string();
+        let cached_resource_value_maybe = get_cached_resource_value_if_valid(
+            db,
+            &authed_client,
+            &job.id,
+            &job.workspace_id,
+            &cached_res_path,
+        )
+        .await;
+        if let Some(cached_resource_value) = cached_resource_value_maybe {
+            let logs = "Job skipped because args & path found in cache and not expired".to_string();
+            job_completed_tx
+                .send(JobCompleted {
+                    job: job,
+                    result: cached_resource_value,
+                    logs,
+                    mem_peak: 0,
+                    canceled_by: None,
+                    success: true,
+                    cached_res_path: None,
+                    token: authed_client.token,
+                })
+                .await
+                .expect("send job completed");
 
-                job_completed_tx
-                    .send(JobCompleted {
-                        job: job,
-                        result,
-                        logs,
-                        mem_peak: 0,
-                        canceled_by: None,
-                        success: true,
-                        cached_res_path: None,
-                        token: authed_client.token,
-                    })
-                    .await
-                    .expect("send job completed");
-
-                return Ok(());
-            }
+            return Ok(());
         }
     };
     match job.job_kind {
-        JobKind::FlowPreview | JobKind::Flow => {
+        JobKind::FlowPreview | JobKind::Flow | JobKind::SingleScriptFlow => {
             let timer = worker_flow_initial_transition_duration.map(|x| x.start_timer());
             handle_flow(
                 &job,
@@ -2464,7 +2556,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                 same_worker_tx,
                 worker_dir,
                 rsmq,
-                worker_name,
+                job_completed_tx.0.clone(),
             )
             .await?;
             timer.map(|x| x.stop_and_record());
@@ -2512,6 +2604,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                         worker_dir,
                         base_internal_url,
                         &client.get_token().await,
+                        rsmq.clone(),
                     )
                     .await
                 }
@@ -2526,6 +2619,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                     worker_dir,
                     base_internal_url,
                     &client.get_token().await,
+                    rsmq.clone(),
                 )
                 .await
                 .map(|()| serde_json::from_str("{}").unwrap()),
@@ -2540,6 +2634,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                     worker_dir,
                     base_internal_url,
                     &client.get_token().await,
+                    rsmq.clone(),
                 )
                 .await
                 .map(|()| serde_json::from_str("{}").unwrap()),
@@ -3041,7 +3136,7 @@ mount {{
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
-async fn handle_dependency_job(
+async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     job: &QueuedJob,
     logs: &mut String,
     mem_peak: &mut i32,
@@ -3052,7 +3147,17 @@ async fn handle_dependency_job(
     worker_dir: &str,
     base_internal_url: &str,
     token: &str,
+    rsmq: Option<R>,
 ) -> error::Result<Box<RawValue>> {
+    let raw_code = sqlx::query_scalar!(
+        "SELECT content FROM script WHERE hash = $1 AND workspace_id = $2",
+        &job.script_hash.unwrap_or(ScriptHash(0)).0,
+        &job.workspace_id
+    )
+    .fetch_optional(db)
+    .await?
+    .unwrap_or_else(|| "No script found at this hash".to_string());
+    let script_path = job.script_path();
     let content = capture_dependency_job(
         &job.id,
         job.language.as_ref().map(|v| Ok(v)).unwrap_or_else(|| {
@@ -3060,10 +3165,7 @@ async fn handle_dependency_job(
                 "Job Language required for dependency jobs".to_owned(),
             ))
         })?,
-        job.raw_code
-            .as_ref()
-            .map(|a| a.as_str())
-            .unwrap_or_else(|| "no raw code"),
+        &raw_code,
         logs,
         mem_peak,
         canceled_by,
@@ -3074,19 +3176,83 @@ async fn handle_dependency_job(
         worker_dir,
         base_internal_url,
         token,
-        job.script_path(),
+        script_path,
     )
     .await;
     match content {
         Ok(content) => {
+            let hash = job.script_hash.unwrap_or(ScriptHash(0));
+            let w_id = &job.workspace_id;
             sqlx::query!(
                 "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
                 &content,
-                &job.script_hash.unwrap_or(ScriptHash(0)).0,
-                &job.workspace_id
+                &hash.0,
+                w_id
             )
             .execute(db)
             .await?;
+
+            let deployment_message = job.raw_code.as_ref().map(|x| x.clone());
+
+            if let Err(e) = handle_deployment_metadata(
+                &job.email,
+                &job.created_by,
+                &db,
+                &w_id,
+                DeployedObject::Script { hash, path: script_path.to_string() },
+                deployment_message.clone(),
+                rsmq.clone(),
+            )
+            .await
+            {
+                tracing::error!(%e, "error handling deployment metadata");
+            }
+
+            if &job.language == &Some(ScriptLang::Python3) {
+                if let Ok(relative_imports) = parse_relative_imports(&raw_code, script_path) {
+                    logs.push_str("\n--- RELATIVE IMPORTS ---\n\n");
+                    logs.push_str(&relative_imports.join("\n"));
+                    if !relative_imports.is_empty() {
+                        let mut tx = db.begin().await?;
+                        sqlx::query!(
+                            "DELETE FROM dependency_map
+                             WHERE importer_path = $1 AND importer_kind = 'script'
+                             AND workspace_id = $2",
+                            script_path,
+                            w_id
+                        )
+                        .execute(&mut *tx)
+                        .await?;
+                        for import in relative_imports {
+                            sqlx::query!(
+                                "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path)
+                                 VALUES ($1, $2, 'script', $3)",
+                                w_id,
+                                script_path,
+                                import
+                            )
+                            .execute(&mut *tx)
+                            .await?;
+                            logs.push_str(&format!("{}\n", import));
+                        }
+                        tx.commit().await?;
+                    }
+                    if let Err(e) = trigger_python_dependents_to_recompute_dependencies(
+                        w_id,
+                        script_path,
+                        deployment_message,
+                        &job.email,
+                        &job.created_by,
+                        &job.permissioned_as,
+                        db,
+                        rsmq,
+                    )
+                    .await
+                    {
+                        tracing::error!(%e, "error triggering python dependents to recompute dependencies");
+                    }
+                }
+            }
             Ok(to_raw_value_owned(
                 json!({ "success": "Successful lock file generation", "lock": content }),
             ))
@@ -3105,7 +3271,80 @@ async fn handle_dependency_job(
     }
 }
 
-async fn handle_flow_dependency_job(
+async fn trigger_python_dependents_to_recompute_dependencies<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
+    w_id: &str,
+    script_path: &str,
+    deployment_message: Option<String>,
+    email: &str,
+    created_by: &str,
+    permissioned_as: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    rsmq: Option<R>,
+) -> error::Result<()> {
+    let script_importers = sqlx::query_scalar!(
+        "SELECT importer_path FROM dependency_map
+         WHERE imported_path = $1 AND importer_kind = 'script'
+         AND workspace_id = $2",
+        script_path,
+        w_id
+    )
+    .fetch_all(db)
+    .await?;
+    for s in script_importers.iter() {
+        let tx: PushIsolationLevel<'_, R> =
+            PushIsolationLevel::IsolatedRoot(db.clone(), rsmq.clone());
+        let r = get_latest_deployed_hash_for_path(db, w_id, s.as_str()).await;
+        if let Ok(r) = r {
+            let (job_uuid, new_tx) = windmill_queue::push(
+                db,
+                tx,
+                &w_id,
+                JobPayload::Dependencies {
+                    path: s.clone(),
+                    hash: r.0,
+                    language: r.5,
+                    dedicated_worker: r.6,
+                    deployment_message: deployment_message.clone(),
+                },
+                PushArgs::empty(),
+                &created_by,
+                email,
+                permissioned_as.to_string(),
+                None,
+                None,
+                None,
+                None,
+                None,
+                false,
+                false,
+                None,
+                true,
+                None,
+                None,
+                None,
+                None,
+            )
+            .await?;
+            tracing::info!(
+                "pushed dependency job due to common python path: {job_uuid} for path {path} with hash {hash}",
+                path = s,
+                hash = r.0
+            );
+            new_tx.commit().await?;
+        } else {
+            tracing::error!(
+                "error getting latest deployed hash for path {path}: {err}",
+                path = s,
+                err = r.unwrap_err()
+            );
+        }
+    }
+    Ok(())
+}
+
+async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     job: &QueuedJob,
     logs: &mut String,
     mem_peak: &mut i32,
@@ -3116,6 +3355,7 @@ async fn handle_flow_dependency_job(
     worker_dir: &str,
     base_internal_url: &str,
     token: &str,
+    rsmq: Option<R>,
 ) -> error::Result<()> {
     let job_path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
@@ -3167,6 +3407,21 @@ async fn handle_flow_dependency_job(
     )
     .execute(db)
     .await?;
+
+    if let Err(e) = handle_deployment_metadata(
+        &job.email,
+        &job.created_by,
+        &db,
+        &job.workspace_id,
+        DeployedObject::Flow { path: job_path },
+        job.raw_code.as_ref().map(|x| x.clone()),
+        rsmq.clone(),
+    )
+    .await
+    {
+        tracing::error!(%e, "error handling deployment metadata");
+    }
+
     Ok(())
 }
 
@@ -3292,22 +3547,11 @@ async fn lock_modules(
             new_flow_modules.push(e);
             continue;
         };
-        // sync with windmill-api/scripts
-        let dependencies = match language {
-            ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(
-                &content,
-                &job.workspace_id,
-                &path.clone().unwrap_or_else(|| job_path.to_string()),
-                &db,
-            )
-            .await?
-            .join("\n"),
-            _ => content.clone(),
-        };
+
         let new_lock = capture_dependency_job(
             &job.id,
             &language,
-            &dependencies,
+            &content,
             logs,
             mem_peak,
             canceled_by,
@@ -3318,7 +3562,7 @@ async fn lock_modules(
             worker_dir,
             base_internal_url,
             token,
-            job.script_path(),
+            &path.clone().unwrap_or_else(|| job_path.to_string()),
         )
         .await;
         match new_lock {
@@ -3393,24 +3637,11 @@ async fn lock_modules_app(
                                 .as_str()
                                 .unwrap_or_default()
                                 .to_string();
-                            let dependencies = match language {
-                                ScriptLang::Python3 => {
-                                    windmill_parser_py_imports::parse_python_imports(
-                                        &content,
-                                        &job.workspace_id,
-                                        job_path,
-                                        &db,
-                                    )
-                                    .await?
-                                    .join("\n")
-                                }
-                                _ => content.clone(),
-                            };
                             logs.push_str("Found lockable inline script. Generating lock...\n");
                             let new_lock = capture_dependency_job(
                                 &job.id,
                                 &language,
-                                &dependencies,
+                                &content,
                                 logs,
                                 mem_peak,
                                 canceled_by,
@@ -3495,7 +3726,7 @@ async fn lock_modules_app(
     }
 }
 
-async fn handle_app_dependency_job(
+async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     job: &QueuedJob,
     logs: &mut String,
     mem_peak: &mut i32,
@@ -3506,6 +3737,7 @@ async fn handle_app_dependency_job(
     worker_dir: &str,
     base_internal_url: &str,
     token: &str,
+    rsmq: Option<R>,
 ) -> error::Result<()> {
     let job_path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
@@ -3555,6 +3787,41 @@ async fn handle_app_dependency_job(
         sqlx::query!("UPDATE app_version SET value = $1 WHERE id = $2", value, id,)
             .execute(db)
             .await?;
+
+        if let Err(e) = handle_deployment_metadata(
+            &job.email,
+            &job.created_by,
+            &db,
+            &job.workspace_id,
+            DeployedObject::App { path: job_path, version: id },
+            job.raw_code.as_ref().map(|x| x.clone()),
+            rsmq.clone(),
+        )
+        .await
+        {
+            tracing::error!(%e, "error handling deployment metadata");
+        }
+
+        // tx = PushIsolationLevel::Transaction(new_tx);
+        // tx = handle_deployment_metadata(
+        //     tx,
+        //     &authed,
+        //     &db,
+        //     &w_id,
+        //     DeployedObject::App { path: app.path.clone(), version: v_id },
+        //     app.deployment_message,
+        // )
+        // .await?;
+
+        // match tx {
+        //     PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        //     _ => {
+        //         return Err(Error::InternalErr(
+        //             "Expected a transaction here".to_string(),
+        //         ));
+        //     }
+        // }
+
         Ok(())
     } else {
         Ok(())
@@ -3579,10 +3846,18 @@ async fn capture_dependency_job(
 ) -> error::Result<String> {
     match job_language {
         ScriptLang::Python3 => {
+            let reqs = windmill_parser_py_imports::parse_python_imports(
+                job_raw_code,
+                &w_id,
+                script_path,
+                &db,
+            )
+            .await?
+            .join("\n");
             create_dependencies_dir(job_dir).await;
             let req: std::result::Result<String, Error> = pip_compile(
                 job_id,
-                job_raw_code,
+                &reqs,
                 logs,
                 mem_peak,
                 canceled_by,

backend/windmill-worker/src/worker_flow.rs

@@ -15,9 +15,8 @@ use std::time::Duration;
 
 use crate::common::{hash_args, save_in_cache};
 use crate::js_eval::{eval_timeout, IdContext};
-use crate::{AuthedClient, PreviousResult, KEEP_JOB_DIR};
+use crate::{AuthedClient, PreviousResult, SendResult, KEEP_JOB_DIR};
 use anyhow::Context;
-use async_recursion::async_recursion;
 use serde::Serialize;
 use serde_json::value::RawValue;
 use serde_json::{json, Value};
@@ -70,8 +69,10 @@ pub async fn update_flow_status_after_job_completion<
     stop_early_override: Option<bool>,
     rsmq: Option<R>,
     worker_name: &str,
+    job_completed_tx: Sender<SendResult>,
 ) -> error::Result<()> {
     // this is manual tailrecursion because async_recursion blows up the stack
+    // todo!();
     let mut rec = update_flow_status_after_job_completion_internal(
         db,
         client,
@@ -87,6 +88,7 @@ pub async fn update_flow_status_after_job_completion<
         false,
         rsmq.clone(),
         worker_name,
+        job_completed_tx.clone(),
     )
     .await?;
     while let Some(nrec) = rec {
@@ -105,6 +107,7 @@ pub async fn update_flow_status_after_job_completion<
             nrec.skip_error_handler,
             rsmq.clone(),
             worker_name,
+            job_completed_tx.clone(),
         )
         .await
         {
@@ -125,6 +128,7 @@ pub async fn update_flow_status_after_job_completion<
                     nrec.skip_error_handler,
                     rsmq.clone(),
                     worker_name,
+                    job_completed_tx.clone(),
                 )
                 .await?
             }
@@ -167,6 +171,7 @@ pub async fn update_flow_status_after_job_completion_internal<
     skip_error_handler: bool,
     rsmq: Option<R>,
     worker_name: &str,
+    job_completed_tx: Sender<SendResult>,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
     let (
         should_continue_flow,
@@ -575,6 +580,8 @@ pub async fn update_flow_status_after_job_completion_internal<
         }
 
         tx.commit().await?;
+        tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated");
+
         (
             should_continue_flow,
             flow_job,
@@ -633,7 +640,8 @@ pub async fn update_flow_status_after_job_completion_internal<
         } else {
             if flow_job.cache_ttl.is_some() {
                 let cached_res_path = {
-                    let args_hash = hash_args(&flow_job.args);
+                    let args_hash =
+                        hash_args(db, client, w_id, job_id_for_status, &flow_job.args).await;
                     let flow_path = flow_job.script_path();
                     let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() {
                         use std::hash::Hasher;
@@ -648,7 +656,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                     format!("{flow_path}/cache/{version_hash}/{args_hash}")
                 };
 
-                save_in_cache(db, &flow_job, cached_res_path, &nresult).await;
+                save_in_cache(db, client, &flow_job, cached_res_path, &nresult).await;
             }
             let success = success && !is_failure_step && !skip_error_handler;
             if success {
@@ -685,6 +693,7 @@ pub async fn update_flow_status_after_job_completion_internal<
         }
         true
     } else {
+        tracing::debug!(id = %flow_job.id,  "start handle flow");
         match handle_flow(
             &flow_job,
             db,
@@ -693,7 +702,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             same_worker_tx.clone(),
             worker_dir,
             rsmq.clone(),
-            worker_name,
+            job_completed_tx,
         )
         .await
         {
@@ -1036,7 +1045,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     rsmq: Option<R>,
-    worker_name: &str,
+    job_completed_tx: Sender<SendResult>,
 ) -> anyhow::Result<()> {
     let flow = flow_job
         .parse_raw_flow()
@@ -1055,7 +1064,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         same_worker_tx,
         worker_dir,
         rsmq,
-        worker_name,
+        job_completed_tx,
     )
     .await?;
     Ok(())
@@ -1088,7 +1097,7 @@ pub struct RawArgs {
     pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
 }
 
-#[async_recursion]
+// #[async_recursion]
 // #[instrument(level = "trace", skip_all)]
 async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     flow_job: &QueuedJob,
@@ -1100,7 +1109,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     rsmq: Option<R>,
-    worker_name: &str,
+    job_completed_tx: Sender<SendResult>,
 ) -> error::Result<()> {
     let job_root = flow_job
         .root_job
@@ -1121,34 +1130,114 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
 
     // if this is an empty module of if the module has already been completed, successfully, update the parent flow
     if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) {
-        let r;
-        return update_flow_status_after_job_completion(
-            db,
-            client,
-            flow_job.id,
-            &Uuid::nil(),
-            flow_job.workspace_id.as_str(),
-            true,
-            if flow.modules.is_empty() {
-                r = to_raw_value(&flow_job_args);
-                &r
-            } else {
-                // it has to be an empty for loop event
-                serde_json::from_str("[]").unwrap()
-            },
-            true,
-            same_worker_tx,
-            worker_dir,
-            None,
-            rsmq,
-            worker_name,
-        )
-        .await;
+        job_completed_tx
+            .send(SendResult::UpdateFlow {
+                flow: flow_job.id,
+                success: true,
+                result: if flow.modules.is_empty() {
+                    to_raw_value(&flow_job_args)
+                } else {
+                    // it has to be an empty for loop event
+                    serde_json::from_str("[]").unwrap()
+                },
+                stop_early_override: None,
+                w_id: flow_job.workspace_id.clone(),
+                worker_dir: worker_dir.to_string(),
+                token: client.token.clone(),
+            })
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!(
+                    "error sending update flow message to job completed channel: {e}"
+                ))
+            })?;
+        // let r;
+        // return update_flow_status_after_job_completion(
+        //     db,
+        //     client,
+        //     flow_job.id,
+        //     &Uuid::nil(),
+        //     flow_job.workspace_id.as_str(),
+        //     true,
+        //     if flow.modules.is_empty() {
+        //         r = to_raw_value(&flow_job_args);
+        //         &r
+        //     } else {
+        //         // it has to be an empty for loop event
+        //         serde_json::from_str("[]").unwrap()
+        //     },
+        //     true,
+        //     same_worker_tx,
+        //     worker_dir,
+        //     None,
+        //     rsmq,
+        //     worker_name,
+        // )
+        // .await;
+        return Ok(());
     }
 
     let arc_flow_job_args = Arc::new(flow_job_args.clone());
 
     if i == 0 {
+        if !flow_job.is_flow_step && flow_job.schedule_path.is_some() {
+            let no_flow_overlap = sqlx::query_scalar!(
+                "SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2",
+                flow_job.schedule_path.as_ref().unwrap(),
+                flow_job.workspace_id.as_str()
+            )
+            .fetch_one(db)
+            .await?;
+            if no_flow_overlap {
+                let count = sqlx::query_scalar!(
+                    "SELECT count(*) FROM queue WHERE schedule_path = $1 AND workspace_id = $2 AND id != $3",
+                    flow_job.schedule_path.as_ref().unwrap(),
+                    flow_job.workspace_id.as_str(),
+                    flow_job.id
+                ).fetch_one(db).await?.unwrap_or(0);
+                if count > 0 {
+                    job_completed_tx
+                        .send(SendResult::UpdateFlow {
+                            flow: flow_job.id,
+                            success: true,
+                            result: serde_json::from_str(
+                                "\"not allowed to overlap, scheduling next iteration\"",
+                            )
+                            .unwrap(),
+                            stop_early_override: Some(true),
+                            w_id: flow_job.workspace_id.clone(),
+                            worker_dir: worker_dir.to_string(),
+                            token: client.token.clone(),
+                        })
+                        .await
+                        .map_err(|e| {
+                            Error::InternalErr(format!(
+                                "error sending update flow message to job completed channel: {e}"
+                            ))
+                        })?;
+                    // return update_flow_status_after_job_completion(
+                    //     db,
+                    //     client,
+                    //     flow_job.id,
+                    //     &Uuid::nil(),
+                    //     flow_job.workspace_id.as_str(),
+                    //     true,
+                    //     serde_json::from_str(
+                    //         "\"not allowed to overlap, scheduling next iteration\"",
+                    //     )
+                    //     .unwrap(),
+                    //     true,
+                    //     same_worker_tx,
+                    //     worker_dir,
+                    //     Some(true),
+                    //     rsmq,
+                    //     worker_name,
+                    // )
+                    // .await;
+                    return Ok(());
+                }
+            }
+        }
         if let Some(skip_expr) = &flow.skip_expr {
             let skip = compute_bool_from_expr(
                 skip_expr.to_string(),
@@ -1160,22 +1249,40 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             )
             .await?;
             if skip {
-                return update_flow_status_after_job_completion(
-                    db,
-                    client,
-                    flow_job.id,
-                    &Uuid::nil(),
-                    flow_job.workspace_id.as_str(),
-                    true,
-                    serde_json::from_str("\"stopped early\"").unwrap(),
-                    true,
-                    same_worker_tx,
-                    worker_dir,
-                    Some(true),
-                    rsmq,
-                    worker_name,
-                )
-                .await;
+                job_completed_tx
+                    .send(SendResult::UpdateFlow {
+                        flow: flow_job.id,
+                        success: true,
+                        result: serde_json::from_str("\"stopped early\"").unwrap(),
+                        stop_early_override: Some(true),
+                        w_id: flow_job.workspace_id.clone(),
+                        worker_dir: worker_dir.to_string(),
+                        token: client.token.clone(),
+                    })
+                    .await
+                    .map_err(|e| {
+                        Error::InternalErr(format!(
+                            "error sending update flow message to job completed channel: {e}"
+                        ))
+                    })?;
+
+                // return update_flow_status_after_job_completion(
+                //     db,
+                //     client,
+                //     flow_job.id,
+                //     &Uuid::nil(),
+                //     flow_job.workspace_id.as_str(),
+                //     true,
+                //     serde_json::from_str("\"stopped early\"").unwrap(),
+                //     true,
+                //     same_worker_tx,
+                //     worker_dir,
+                //     Some(true),
+                //     rsmq,
+                //     worker_name,
+                // )
+                // .await;
+                return Ok(());
             }
         }
     }
@@ -1397,22 +1504,39 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 .await?;
                 if flow_job.is_flow_step {
                     if let Some(parent_job) = flow_job.parent_job {
-                        update_flow_status_after_job_completion(
-                            db,
-                            client,
-                            parent_job,
-                            &flow_job.id,
-                            &flow_job.workspace_id,
-                            true,
-                            &to_raw_value(&result),
-                            false,
-                            same_worker_tx.clone(),
-                            &worker_dir,
-                            None,
-                            rsmq,
-                            worker_name,
-                        )
-                        .await?;
+                        job_completed_tx
+                            .send(SendResult::UpdateFlow {
+                                flow: parent_job,
+                                success: true,
+                                result: to_raw_value(&result),
+                                stop_early_override: Some(true),
+                                w_id: flow_job.workspace_id.clone(),
+                                worker_dir: worker_dir.to_string(),
+                                token: client.token.clone(),
+                            })
+                            .await
+                            .map_err(|e| {
+                                Error::InternalErr(format!(
+                                "error sending update flow message to job completed channel: {e}"
+                            ))
+                            })?;
+                        // update_flow_status_after_job_completion(
+                        //     db,
+                        //     client,
+                        //     parent_job,
+                        //     &flow_job.id,
+                        //     &flow_job.workspace_id,
+                        //     true,
+                        //     &to_raw_value(&result),
+                        //     false,
+                        //     same_worker_tx.clone(),
+                        //     &worker_dir,
+                        //     None,
+                        //     rsmq,
+                        //     worker_name,
+                        // )
+                        // .await?;
+                        return Ok(());
                     }
                 }
                 return Ok(());
@@ -1650,6 +1774,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             _ => Ok(flow_job_args),
         }
     };
+    tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow job args computed");
 
     let next_flow_transform = compute_next_flow_transform(
         arc_flow_job_args.clone(),
@@ -1850,6 +1975,8 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         )
         .await?;
 
+        tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}");
+
         if let FlowModuleValue::ForloopFlow { parallelism: Some(p), .. } = &module.value {
             if i as u16 >= *p {
                 sqlx::query!(
@@ -2172,6 +2299,8 @@ async fn compute_next_flow_transform(
         ))
     };
     let delete_after_use = module.delete_after_use.unwrap_or(false);
+
+    tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value);
     match &module.value {
         FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
         FlowModuleValue::Flow { path, .. } => {