chore(main): release 1.167.0 (#2227)

* chore(main): release 1.167.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/Dockerfile

@@ -1,7 +1,7 @@
 FROM nikolaik/python-nodejs
 
 RUN npm install -g @apidevtools/swagger-cli
-RUN pip install openapi-python-client
+RUN pip install openapi-python-client==0.15.1
 RUN pip install poetry
 
 

.github/DockerfileBackendTests

@@ -33,7 +33,7 @@ ENV SQLX_OFFLINE=true
 
 RUN mkdir -p /frontend/build
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
+    && apt-get install -y ca-certificates tzdata libpq5 cmake\
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \

.github/DockerfilePypiBuilder

@@ -4,4 +4,4 @@ RUN python3 -m pip install pipx poetry
 RUN python3 -m pipx ensurepath
 ENV PATH="/root/.local/bin:${PATH}"
 ENV PATH="/usr/local/bin:${PATH}"
-RUN pipx install openapi-python-client==0.11.6 --include-deps
+RUN pipx install openapi-python-client==0.15.1 --include-deps

.github/workflows/backend-test.yml

@@ -18,7 +18,6 @@ jobs:
     runs-on: [self-hosted, new]
     container:
       image: ghcr.io/windmill-labs/backend-tests
-      options: --privileged
     services:
       postgres:
         image: postgres
@@ -27,9 +26,7 @@ jobs:
           POSTGRES_PASSWORD: changeme
 
         options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
     steps:
       - uses: actions/checkout@v3
@@ -41,4 +38,8 @@ jobs:
             backend -> target
       - name: cargo test
         timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        run:
+          mkdir frontend/build && cd backend && touch
+          windmill-api/openapi-deref.yaml &&
+          DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
+          test --all -- --nocapture

.github/workflows/benchmark.yml

@@ -0,0 +1,41 @@
+name: Run benchmarks
+
+on:
+  schedule:
+    - cron: "0 0 */1 * *"
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    runs-on: [self-hosted, new]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+      windmill:
+        image: ghcr.io/windmill-labs/windmill:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+    steps:
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+      - name: benchmark
+        timeout-minutes: 10
+        run:
+          deno run --unstable -A
+          https://raw.githubusercontent.com/windmill-labs/windmill/main/benchmarks/benchmark_noop.ts
+          --host http://localhost:8000 -e admin@windmill.dev -p changeme -j
+          10000

.github/workflows/docker-mssql.yml

@@ -3,7 +3,10 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 name: Build windmill:mssql
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 */4 * *"
 
 concurrency:
   group: ${{ github.ref }}-mssql

CHANGELOG.md

@@ -1,6 +1,275 @@
 # Changelog
 
 
+## [1.167.0](https://github.com/windmill-labs/windmill/compare/v1.166.1...v1.167.0) (2023-09-04)
+
+
+### Features
+
+* submit result in background thread (unify architecture for dedicated worker) ([#2226](https://github.com/windmill-labs/windmill/issues/2226)) ([dff1cd9](https://github.com/windmill-labs/windmill/commit/dff1cd9a64f755f239eb57599c104c47f4d33b12))
+
+
+### Bug Fixes
+
+* **cli:** prioritize correctly content file to resolve for ts types ([2906d53](https://github.com/windmill-labs/windmill/commit/2906d535a126f4fe2cfe6dffda46e5fe841056da))
+
+## [1.166.1](https://github.com/windmill-labs/windmill/compare/v1.166.0...v1.166.1) (2023-09-03)
+
+
+### Bug Fixes
+
+* fix setting is ready for s3 workers ([b0ed0f9](https://github.com/windmill-labs/windmill/commit/b0ed0f964843247d11ecfe586f1565589df95ff6))
+
+## [1.166.0](https://github.com/windmill-labs/windmill/compare/v1.165.0...v1.166.0) (2023-09-03)
+
+
+### Features
+
+* **frontend:** App stepper debug ([#2202](https://github.com/windmill-labs/windmill/issues/2202)) ([77f8eac](https://github.com/windmill-labs/windmill/commit/77f8eac21e0edfa1eada617d78a498a3a6ae1dce))
+
+
+### Bug Fixes
+
+* fix datetime handling for python ([b35ffd4](https://github.com/windmill-labs/windmill/commit/b35ffd435de97ed34fcda69490abd734ea3229fa))
+* **frontend:** Fix App Modal z-index ([#2210](https://github.com/windmill-labs/windmill/issues/2210)) ([9787edb](https://github.com/windmill-labs/windmill/commit/9787edb67c329265bf179fe304d00cdc1df7042e))
+* see run detail in a new tab ([719a7b1](https://github.com/windmill-labs/windmill/commit/719a7b11da81f68452ba9fc22ff456fe1ddde1de))
+* update wmill python generator thus updating windmill-api ([f912f1d](https://github.com/windmill-labs/windmill/commit/f912f1de86e91c5cdbc0012e2362467c4965936a))
+
+
+### Performance Improvements
+
+* improve queue performance ([#2222](https://github.com/windmill-labs/windmill/issues/2222)) ([069e2d1](https://github.com/windmill-labs/windmill/commit/069e2d18d586aa3d407e3b089d1ad94b2b838af0))
+
+## [1.165.0](https://github.com/windmill-labs/windmill/compare/v1.164.0...v1.165.0) (2023-08-31)
+
+
+### Features
+
+* improve queue performance when queue grows large ([ada88a2](https://github.com/windmill-labs/windmill/commit/ada88a2bf94fec71187bbdb210065de43d4cd3fb))
+* support partial go dependency pinning ([41107c7](https://github.com/windmill-labs/windmill/commit/41107c7cfa7b56099a9c8b08cfb16ff3cf840ff2))
+
+
+### Bug Fixes
+
+* uniformize that all job links specify the workspace ([d311d76](https://github.com/windmill-labs/windmill/commit/d311d76557432a72a5d6d7ab010aeb1fe0e599de))
+
+## [1.164.0](https://github.com/windmill-labs/windmill/compare/v1.163.1...v1.164.0) (2023-08-31)
+
+
+### Features
+
+* add workspace variable to worker tag ([276cd6d](https://github.com/windmill-labs/windmill/commit/276cd6dac39b7cb181ac46e3edea79a3a3bcff8d))
+
+
+### Bug Fixes
+
+* **frontend:** allow using Docker in Flow ([#2201](https://github.com/windmill-labs/windmill/issues/2201)) ([bb749c1](https://github.com/windmill-labs/windmill/commit/bb749c14f877f7cb1e8642b881a00aedfeb08f7d))
+
+## [1.163.1](https://github.com/windmill-labs/windmill/compare/v1.163.0...v1.163.1) (2023-08-30)
+
+
+### Bug Fixes
+
+* avoid perpetual spinning of recompute all component ([11e1ecb](https://github.com/windmill-labs/windmill/commit/11e1ecbcda92f5ab643b776094ef10005d51b579))
+
+## [1.163.0](https://github.com/windmill-labs/windmill/compare/v1.162.2...v1.163.0) (2023-08-30)
+
+
+### Features
+
+* add global cache configuration ([7c5ea56](https://github.com/windmill-labs/windmill/commit/7c5ea569a8102ef052d42216e2ff8d4c3169a7a5))
+
+
+### Bug Fixes
+
+* fix cyclical loop in apps ([61df339](https://github.com/windmill-labs/windmill/commit/61df339343767e63cbe7a4e75f1fd4f848dbd7e0))
+
+## [1.162.2](https://github.com/windmill-labs/windmill/compare/v1.162.1...v1.162.2) (2023-08-29)
+
+
+### Bug Fixes
+
+* fix incorrect bump ([4704899](https://github.com/windmill-labs/windmill/commit/4704899a81cb281b99949c934184e23b199b2ed8))
+
+## [1.162.1](https://github.com/windmill-labs/windmill/compare/v1.162.0...v1.162.1) (2023-08-29)
+
+
+### Bug Fixes
+
+* fix deps incompatibilities ([6c5a8a3](https://github.com/windmill-labs/windmill/commit/6c5a8a3613b4608e6d2b57e7f40cd4ab2d1af9ae))
+
+## [1.162.0](https://github.com/windmill-labs/windmill/compare/v1.161.0...v1.162.0) (2023-08-29)
+
+
+### Features
+
+* add cache to inline scripts ([bf0014c](https://github.com/windmill-labs/windmill/commit/bf0014c387361ce358d31c7cbc44a9c4c97606df))
+* add caching to flows and scripts ([#2193](https://github.com/windmill-labs/windmill/issues/2193)) ([03e48a4](https://github.com/windmill-labs/windmill/commit/03e48a4ca557cd2c385988d3a935cea38bc6e81e))
+* **frontend:** Filter runs by user ([#2187](https://github.com/windmill-labs/windmill/issues/2187)) ([095969f](https://github.com/windmill-labs/windmill/commit/095969f125e9186cb4f02f75e914ef9a70e3abc4))
+
+
+### Bug Fixes
+
+* add setState, getState to client ([67f868f](https://github.com/windmill-labs/windmill/commit/67f868f08ed10f3f7c185af67bff7080c339e974))
+* relative imports in deno ([30ea354](https://github.com/windmill-labs/windmill/commit/30ea354cae91ea040b3112c4138a1e5f0d7ab530))
+
+## [1.161.0](https://github.com/windmill-labs/windmill/compare/v1.160.0...v1.161.0) (2023-08-28)
+
+
+### Features
+
+* concurrency limits for flows ([d0d041f](https://github.com/windmill-labs/windmill/commit/d0d041fde37ceda5e3a04e5da9c87d6b7e5691b3))
+* early stop for flows ([6354c95](https://github.com/windmill-labs/windmill/commit/6354c95bb74c5d1af838234c0146176a0d3e408e))
+* **frontend:** rework premium plans ([#2155](https://github.com/windmill-labs/windmill/issues/2155)) ([272ff63](https://github.com/windmill-labs/windmill/commit/272ff63e4072b4c25a46c133b518649f88b7598e))
+
+
+### Bug Fixes
+
+* allow deno to --write lock when using lockfiles ([770a3e8](https://github.com/windmill-labs/windmill/commit/770a3e8835637af1b1e017ecc1675e526ca40345))
+* fix refresh init in presence of app stepper ([840fbbc](https://github.com/windmill-labs/windmill/commit/840fbbcbb1f969ef3b000f9e50d5c5dde8371995))
+
+## [1.160.0](https://github.com/windmill-labs/windmill/compare/v1.159.0...v1.160.0) (2023-08-27)
+
+
+### Features
+
+* add parallelism control to forloops ([34e2a80](https://github.com/windmill-labs/windmill/commit/34e2a8001afa8bb948bf907383bffbc8aa11901f))
+
+## [1.159.0](https://github.com/windmill-labs/windmill/compare/v1.158.2...v1.159.0) (2023-08-27)
+
+
+### Features
+
+* add support for root certificate in postgresql ([b492fd9](https://github.com/windmill-labs/windmill/commit/b492fd98846ff4b4e073bb41de91dd84f0bd7031))
+* support to set linked secret variable to any field of a newly created resource ([fe1e419](https://github.com/windmill-labs/windmill/commit/fe1e419fa83db6a9db59aac23490e52cd3649f51))
+
+
+### Bug Fixes
+
+* canceling jobs ([0dfdf8f](https://github.com/windmill-labs/windmill/commit/0dfdf8fa1be88d601f7dbf7b348aaf8a3ae8e2fd))
+* fix app table footer label when -1 ([24ac1e2](https://github.com/windmill-labs/windmill/commit/24ac1e25ff87eef591e9f766bd0e7991b3668723))
+* operation are redacted instead of username which fix audit logs for non admin users ([487d56c](https://github.com/windmill-labs/windmill/commit/487d56cb0fedde47c77cdb7a4b5424b51c4a2e10))
+
+## [1.158.2](https://github.com/windmill-labs/windmill/compare/v1.158.1...v1.158.2) (2023-08-26)
+
+
+### Bug Fixes
+
+* expose getResumeUrls in windmill-client ([3142bc9](https://github.com/windmill-labs/windmill/commit/3142bc932c8ca915b9dda8879d31ef19ecfaa07f))
+
+## [1.158.1](https://github.com/windmill-labs/windmill/compare/v1.158.0...v1.158.1) (2023-08-26)
+
+
+### Bug Fixes
+
+* fix windmill-client ([7defd45](https://github.com/windmill-labs/windmill/commit/7defd451ac847b9824d503d0b7685344221ff564))
+
+## [1.158.0](https://github.com/windmill-labs/windmill/compare/v1.157.0...v1.158.0) (2023-08-26)
+
+
+### Features
+
+* add lockfile for deno + use npm module for deno for windmill-client ([9547a06](https://github.com/windmill-labs/windmill/commit/9547a061da0b80a4bc278ee09a0004d410ec7410))
+
+## [1.157.0](https://github.com/windmill-labs/windmill/compare/v1.156.1...v1.157.0) (2023-08-26)
+
+
+### Features
+
+* lock inline scripts for apps on deploy ([f5121e9](https://github.com/windmill-labs/windmill/commit/f5121e9066e1a93ad6f928daad891a08ae840d81))
+
+
+### Bug Fixes
+
+* make workspace error handler picker accept any script ([53976da](https://github.com/windmill-labs/windmill/commit/53976da8ae70de3f8e251564220312541604d77b))
+
+## [1.156.1](https://github.com/windmill-labs/windmill/compare/v1.156.0...v1.156.1) (2023-08-25)
+
+
+### Bug Fixes
+
+* fix python client ([7649a53](https://github.com/windmill-labs/windmill/commit/7649a53f3c792ceba8f2a0fc8535c512b25bf969))
+
+## [1.156.0](https://github.com/windmill-labs/windmill/compare/v1.155.0...v1.156.0) (2023-08-24)
+
+
+### Features
+
+* schedule recovery handler ([#2126](https://github.com/windmill-labs/windmill/issues/2126)) ([0dcb425](https://github.com/windmill-labs/windmill/commit/0dcb425e4a9cf241ed301f794680b36a7f17cc34))
+
+## [1.155.0](https://github.com/windmill-labs/windmill/compare/v1.154.2...v1.155.0) (2023-08-24)
+
+
+### Features
+
+* add templatev2 using new eval ([13d870f](https://github.com/windmill-labs/windmill/commit/13d870f16370a74fe481a1701eda27109a776c75))
+* eval v2, blazing fast eval triggered only upon the right changes ([#2164](https://github.com/windmill-labs/windmill/issues/2164)) ([5207a7a](https://github.com/windmill-labs/windmill/commit/5207a7a6aa1520c987d26d5c1f99f653c1c81cf6))
+* remove connect in favor of eval ([e7aaa17](https://github.com/windmill-labs/windmill/commit/e7aaa177b72749ca9d0d78c452ec8e47d6514186))
+
+
+### Bug Fixes
+
+* bump bun to 0.8.0 ([4825519](https://github.com/windmill-labs/windmill/commit/4825519ac94a4992cf21fbf4a21fbea8038058d9))
+* fix tables not updating inputs on creation ([a419bc4](https://github.com/windmill-labs/windmill/commit/a419bc41bfadce1ac75383d1824ff9fef3404aad))
+* **frontend:** Fix code display + use async/await in fetch examples ([#2150](https://github.com/windmill-labs/windmill/issues/2150)) ([2f9177f](https://github.com/windmill-labs/windmill/commit/2f9177f6cec0a676c774ee426482f55227e6e388))
+* **frontend:** fix copyToClipboard on non-HTTPS site ([#2046](https://github.com/windmill-labs/windmill/issues/2046)) ([95ea0e8](https://github.com/windmill-labs/windmill/commit/95ea0e8f87195816dde3f9554b3cb92791b63a37))
+* update go to 1.12.0 and deno to 1.36.2 ([4317573](https://github.com/windmill-labs/windmill/commit/431757339bbfff6d67f484439d87255acc5c62ff))
+* update python client with by_path methods ([8a25a86](https://github.com/windmill-labs/windmill/commit/8a25a86e586485e7949bb208fa94db906e983b6c))
+
+## [1.154.2](https://github.com/windmill-labs/windmill/compare/v1.154.1...v1.154.2) (2023-08-22)
+
+
+### Bug Fixes
+
+* fix cancel job for flows in some edge cases ([58bb19a](https://github.com/windmill-labs/windmill/commit/58bb19a4471ce8cfced4b144fca40069b5ce0820))
+
+## [1.154.1](https://github.com/windmill-labs/windmill/compare/v1.154.0...v1.154.1) (2023-08-22)
+
+
+### Bug Fixes
+
+* **frontend:** Fix hub navigation ([#2151](https://github.com/windmill-labs/windmill/issues/2151)) ([d0ed8f0](https://github.com/windmill-labs/windmill/commit/d0ed8f0fefe3176b9bab621a6b3e9231254504e2))
+* show for-loop settings ([ab8a27f](https://github.com/windmill-labs/windmill/commit/ab8a27f123fbca187eee3b372d512797f8a03916))
+
+## [1.154.0](https://github.com/windmill-labs/windmill/compare/v1.153.0...v1.154.0) (2023-08-21)
+
+
+### Features
+
+* deploy folders as well in the UI deployer ([bcf5d4e](https://github.com/windmill-labs/windmill/commit/bcf5d4e5d42a7d17e2d1932b030cca101d9de9b4))
+
+
+### Bug Fixes
+
+* avoid stack-overflow on jsruntime for recursive objects ([127eea3](https://github.com/windmill-labs/windmill/commit/127eea3c8144b14b8f78a196f5c2cd245d2caad9))
+* do not require auth for OPTIONS requests ([bdd59c9](https://github.com/windmill-labs/windmill/commit/bdd59c94a9bde10e808427ef529d1b6ab6e78a45))
+
+## [1.153.0](https://github.com/windmill-labs/windmill/compare/v1.152.0...v1.153.0) (2023-08-20)
+
+
+### Features
+
+* multiline support in bash ([e1469cc](https://github.com/windmill-labs/windmill/commit/e1469cc64d672b5fc42edac313bc11a017812511))
+
+
+### Bug Fixes
+
+* update deno-client to use new Resource and Variable endpoints ([c13428a](https://github.com/windmill-labs/windmill/commit/c13428ad089999e38768b86bfd251d747759dc69))
+
+## [1.152.0](https://github.com/windmill-labs/windmill/compare/v1.151.2...v1.152.0) (2023-08-20)
+
+
+### Features
+
+* handle drift in every time referencing db times ([b9fb206](https://github.com/windmill-labs/windmill/commit/b9fb206c112798f3776ba0e6da70e86e7c769a1f))
+* prometheus metrics are now ee only ([2afea50](https://github.com/windmill-labs/windmill/commit/2afea504977f9cd08d62c5f85be1fd2cefe8a691))
+
+
+### Bug Fixes
+
+* improve progress bar UX ([85d2d47](https://github.com/windmill-labs/windmill/commit/85d2d4782779d981a131f48db6e1058fe79daeef))
+* reinit retry to undefined in flow steps ([75f4723](https://github.com/windmill-labs/windmill/commit/75f472381cfa73d77295b29a202efbd58c79918d))
+
 ## [1.151.2](https://github.com/windmill-labs/windmill/compare/v1.151.1...v1.151.2) (2023-08-18)
 
 

Dockerfile

@@ -73,7 +73,7 @@ ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN apt-get update && apt-get install -y libxml2-dev libxmlsec1-dev clang libclang-dev
+RUN apt-get update && apt-get install -y libxml2-dev libxmlsec1-dev clang libclang-dev cmake
 
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
 
@@ -96,8 +96,8 @@ SHELL ["/bin/bash", "-c"]
 RUN apt update -y
 RUN apt install -y unzip curl
 
-RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.1/deno-linux-arm64.zip -o deno.zip || true
-RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.1/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.3/deno-linux-arm64.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.3/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
 
 RUN unzip deno.zip && rm deno.zip
 
@@ -149,13 +149,13 @@ RUN set -eux; \
     arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
     case "$arch" in \
     'amd64') \
-    targz='go1.19.3.linux-amd64.tar.gz'; \
+    targz='go1.21.0.linux-amd64.tar.gz'; \
     ;; \
     'arm64') \
-    targz='go1.19.3.linux-arm64.tar.gz'; \
+    targz='go1.21.0.linux-arm64.tar.gz'; \
     ;; \
     'armhf') \
-    targz='go1.19.3.linux-armv6l.tar.gz'; \
+    targz='go1.21.0.linux-armv6l.tar.gz'; \
     ;; \
     *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
     esac; \
@@ -177,7 +177,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:0.7.3 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

README.md

@@ -85,7 +85,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
    shared on [WindmillHub](https://hub.windmill.dev).
    ![Step 3](./imgs/windmill-flow.png)
 
-4. Build complex UI on top of your scripts and flows.
+4. Build complex UIs on top of your scripts and flows.
    ![Step 4](./imgs/windmill-builder.png)
 
 Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
@@ -336,10 +336,11 @@ it being synced automatically everyday.
 | PORT                                          | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
 | NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |
-| METRICS_ADDR                                  | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| METRICS_ADDR                                  | None                                       | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                      | All                   |
 | JSON_FMT                                      | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
 | BASE_URL                                      | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| TIMEOUT                                       | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| TIMEOUT                                       | 60 _ 60 _ 24 \* 7 (1 week)                 | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     |
+| SCRIPT_TOKEN_EXPIRY                           | 900                                        | The default duration period of the ephemeral-token generated at the beginning of a script                                                                                                          | Worker                |
 | ZOMBIE_JOB_TIMEOUT                            | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
 | RESTART_ZOMBIE_JOBS                           | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
 | SLEEP_QUEUE                                   | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |

backend/.gitignore

@@ -1,4 +1,5 @@
 target/
 .env
 oauth.json
-windmill-api/openapi-deref.yaml
+windmill-api/openapi-deref.yaml
+tracing.folded

backend/.sqlx/query-0694f4be1a7966126f20afbd9ee094088db8ce98aefdbe96144c42e38e5bd813.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select tag, concurrent_limit, concurrency_time_window_s from script where hash = $1 AND workspace_id = $2",
+  "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -17,6 +17,11 @@
         "ordinal": 2,
         "name": "concurrency_time_window_s",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "cache_ttl",
+        "type_info": "Int4"
       }
     ],
     "parameters": {
@@ -26,10 +31,11 @@
       ]
     },
     "nullable": [
+      true,
       true,
       true,
       true
     ]
   },
-  "hash": "8f0d98661f2a32659f37a123320d14288ce88c9d6e6279bdc36334410023f5e1"
+  "hash": "0694f4be1a7966126f20afbd9ee094088db8ce98aefdbe96144c42e38e5bd813"
 }

backend/.sqlx/query-077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74.json

@@ -1,29 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2)\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n                FROM queue\n                WHERE script_path = $1 AND running = true\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "min_started_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 1,
-        "name": "total_count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Float8"
-      ]
-    },
-    "nullable": [
-      true,
-      true
-    ]
-  },
-  "hash": "077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74"
-}

backend/.sqlx/query-0be97088e8ca1e8be9bf4ea45f830cceedd598a3221c522a1f9026d2d48a0ffb.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
+  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10 WHERE path = $11 AND workspace_id = $12 RETURNING *",
   "describe": {
     "columns": [
       {
@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -80,6 +110,12 @@
         "Varchar",
         "Jsonb",
         "Varchar",
+        "Int4",
+        "Bool",
+        "Json",
+        "Varchar",
+        "Int4",
+        "Json",
         "Text",
         "Text"
       ]
@@ -98,8 +134,14 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "1e0769c59ada78848e034250c11be0ca2e8c49b96a0066b9ab67f5a205ec456d"
+  "hash": "0be97088e8ca1e8be9bf4ea45f830cceedd598a3221c522a1f9026d2d48a0ffb"
 }

backend/.sqlx/query-18bdd06f4791c652addb4783f419f5d020c1d23278f583c6be6314768f9bf25a.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM group_ WHERE name != 'all'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "18bdd06f4791c652addb4783f419f5d020c1d23278f583c6be6314768f9bf25a"
+}

backend/.sqlx/query-1c1577b9963d907c4245a027fece57285ce64ac41a84681b32deb71e452334c1.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM global_settings WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "1c1577b9963d907c4245a027fece57285ce64ac41a84681b32deb71e452334c1"
+}

backend/.sqlx/query-1e232d4b978d133ba542d0abe5afaa0031f630c5336c0b805590b5ea4508ed04.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "VACUUM queue",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "1e232d4b978d133ba542d0abe5afaa0031f630c5336c0b805590b5ea4508ed04"
+}

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -44,7 +44,8 @@
                 "http",
                 "graphql",
                 "postgresql",
-                "noop"
+                "noop",
+                "appdependencies"
               ]
             }
           }

backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM app_version WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a"
+}

backend/.sqlx/query-330e85c6fe52355971262d3a44c2b75a95b1d042b9e0995186aa5a2d0b7ee552.json

@@ -0,0 +1,30 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) AS completed_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND job_kind != 'dependencies' AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at\n                FROM queue\n                WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "min_started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 1,
+        "name": "completed_count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Float8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true
+    ]
+  },
+  "hash": "330e85c6fe52355971262d3a44c2b75a95b1d042b9e0995186aa5a2d0b7ee552"
+}

backend/.sqlx/query-3c83e6adf21b7a8572529f9175cf6d2d30ea8c921ae2bf751b6825a3e453a012.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                UPDATE queue\n                   SET suspend = $1, suspend_until = now() + interval '14 day', running = true\n                 WHERE id = $2\n                ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3c83e6adf21b7a8572529f9175cf6d2d30ea8c921ae2bf751b6825a3e453a012"
+}

backend/.sqlx/query-3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT  usr.username  \n            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username \n            WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
+  "query": "SELECT  usr.username  \n            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2\n            WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       false
     ]
   },
-  "hash": "c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a"
+  "hash": "3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0"
 }

backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
+  "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -85,5 +85,5 @@
       true
     ]
   },
-  "hash": "7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c"
+  "hash": "41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0"
 }

backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT * FROM resource WHERE workspace_id = $1",
+  "query": "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
   "describe": {
     "columns": [
       {
@@ -48,5 +48,5 @@
       false
     ]
   },
-  "hash": "fa258894bd90ea6586669e5810c5c6bcb42d5e1e68fab27fb185d06962b6454a"
+  "hash": "45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7"
 }

backend/.sqlx/query-467247beca4dae321ab3acc3a99f655412761157cda54bbd3e58cada3de3396c.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "467247beca4dae321ab3acc3a99f655412761157cda54bbd3e58cada3de3396c"
+}

backend/.sqlx/query-4a3917318d967d96c4b6e70786a034302f04da9d1e9eec59b1eb6052c2d947be.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "success",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 1,
+        "name": "result",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 2,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Uuid",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "4a3917318d967d96c4b6e70786a034302f04da9d1e9eec59b1eb6052c2d947be"
+}

backend/.sqlx/query-52dbe77fad5729e9e121609eeb6a3573e6d0e1d1c16e29b15c7ad810536e249c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -57,10 +57,11 @@
         "Bool",
         "VarcharArray",
         "Int4",
+        "Int4",
         "Int4"
       ]
     },
     "nullable": []
   },
-  "hash": "9ee1423945740c17dedc2fc7ea7e7183380c43d0feeb615c651aa486eef3bd53"
+  "hash": "52dbe77fad5729e9e121609eeb6a3573e6d0e1d1c16e29b15c7ad810536e249c"
 }

backend/.sqlx/query-6dbf0275e52d937a114ba305c6646d313f6ef05062ac55f7397caf8c02ccec5e.json

@@ -22,7 +22,8 @@
                 "http",
                 "graphql",
                 "postgresql",
-                "noop"
+                "noop",
+                "appdependencies"
               ]
             }
           }

backend/.sqlx/query-6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE app_version SET value = $1 WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Json",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9"
+}

backend/.sqlx/query-71879f3644dde27f2d93e5511e420579dfd2ffab72bb8480a1622b68703d4ea7.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1 RETURNING counter",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "counter",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "71879f3644dde27f2d93e5511e420579dfd2ffab72bb8480a1622b68703d4ea7"
+}

backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND schedule.workspace_id = completed_job.workspace_id AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -75,6 +75,36 @@
       },
       {
         "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
         "name": "jobs",
         "type_info": "JsonArray"
       }
@@ -101,8 +131,14 @@
       true,
       false,
       true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       null
     ]
   },
-  "hash": "dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7"
+  "hash": "7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097"
 }

backend/.sqlx/query-8062bb1ac475c4a2a016d9dfa63c95765356fe7744b817ad0c320a54a33b24f3.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
+  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
   "describe": {
     "columns": [
       {
@@ -22,6 +22,11 @@
         "ordinal": 3,
         "name": "concurrency_time_window_s",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
       }
     ],
     "parameters": {
@@ -34,8 +39,9 @@
       false,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "7435be9a71a6b30980d180047486a47d69e36132d40ef0f53448e96a9866842b"
+  "hash": "8062bb1ac475c4a2a016d9dfa63c95765356fe7744b817ad0c320a54a33b24f3"
 }

backend/.sqlx/query-86652246b20f9bd023d3419ec382176780a87f32673b53de6f899564c1494147.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO concurrency_counter VALUES ($1, 1)\n        ON CONFLICT (concurrency_id) \n        DO UPDATE SET counter = concurrency_counter.counter + 1\n        RETURNING concurrency_counter.counter",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "counter",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "86652246b20f9bd023d3419ec382176780a87f32673b53de6f899564c1494147"
+}

backend/.sqlx/query-911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -95,6 +125,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-960dce02871e9697d7aca2fcd5a75eefc3db7179e49fbf20c7e62cbcb235683e.json

@@ -1,8 +1,14 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET  canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 ",
+  "query": "UPDATE queue SET  canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 RETURNING id",
   "describe": {
-    "columns": [],
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
     "parameters": {
       "Left": [
         "Varchar",
@@ -11,7 +17,9 @@
         "Text"
       ]
     },
-    "nullable": []
+    "nullable": [
+      false
+    ]
   },
-  "hash": "b201a427430e732c45f57e0ebeba7ff9c2cbbd9f6b15266dcea789586b09e1e8"
+  "hash": "960dce02871e9697d7aca2fcd5a75eefc3db7179e49fbf20c7e62cbcb235683e"
 }

backend/.sqlx/query-96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -96,6 +126,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-9b9afb9d5ef57b70de1207c4546b5547e579d978d899325d821a5aee1df1a9fc.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *",
+  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *",
   "describe": {
     "columns": [
       {
@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -86,7 +116,13 @@
         "Jsonb",
         "Bool",
         "Varchar",
-        "Varchar"
+        "Varchar",
+        "Int4",
+        "Bool",
+        "Json",
+        "Varchar",
+        "Int4",
+        "Json"
       ]
     },
     "nullable": [
@@ -103,8 +139,14 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "80e632cb6618cde2bac2c12d0afc796a4a1768f902d0540af33fe6f442cf583a"
+  "hash": "9b9afb9d5ef57b70de1207c4546b5547e579d978d899325d821a5aee1df1a9fc"
 }

backend/.sqlx/query-9da1bf41c433c1f781bcc8ea5d86fd796b7ff92434f5ed4682c4445a02c2c720.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1",
+  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1 AND scheduled_for <= now() AND running = false",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       null
     ]
   },
-  "hash": "c4fb68ce79c1cec23cadc515285578ca18398504cf703091d06191ff68d95399"
+  "hash": "9da1bf41c433c1f781bcc8ea5d86fd796b7ff92434f5ed4682c4445a02c2c720"
 }

backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO queue\n            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n                script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id)\n            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28) RETURNING id",
+  "query": "INSERT INTO queue\n            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n                script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl)\n            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29) RETURNING id",
   "describe": {
     "columns": [
       {
@@ -39,7 +39,8 @@
                 "http",
                 "graphql",
                 "postgresql",
-                "noop"
+                "noop",
+                "appdependencies"
               ]
             }
           }
@@ -79,12 +80,13 @@
         "Int4",
         "Int4",
         "Int4",
-        "Varchar"
+        "Varchar",
+        "Int4"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "01d5e4e371a507602fca01190965a4f20ff806315a9ed02b372977e2d9479338"
+  "hash": "9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b"
 }

backend/.sqlx/query-a4ae245dcf7e4b930cd45701db0b7c45f2a5797e8b6724bade5b964d4334c098.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET suspend = suspend - 1 WHERE parent_job = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a4ae245dcf7e4b930cd45701db0b7c45f2a5797e8b6724bade5b964d4334c098"
+}

backend/.sqlx/query-a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681"
+}

backend/.sqlx/query-aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -94,6 +124,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n        app.extra_perms, app_version.value, \n        app_version.created_at, app_version.created_by,\n        app.draft_only, draft.value as \"draft?\"\n        from app\n        INNER JOIN app_version ON\n        app_version.id = app.versions[array_upper(app.versions, 1)]\n        LEFT JOIN draft ON \n        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' \n        WHERE app.path = $1 AND app.workspace_id = $2",
+  "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n        app.extra_perms, app_version.value, \n        app_version.created_at, app_version.created_by,\n        app.draft_only, draft.value as \"draft?\"\n        from app\n        INNER JOIN app_version ON\n        app_version.id = app.versions[array_upper(app.versions, 1)]\n        LEFT JOIN draft ON \n        app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' \n        WHERE app.path = $1 AND app.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -79,5 +79,5 @@
       false
     ]
   },
-  "hash": "f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4"
+  "hash": "aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f"
 }

backend/.sqlx/query-c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -95,6 +125,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-cf21b61aa5b5e47744235c860e2d9b90d7654a4b5d0ec32085b178399f701bfc.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND archived = false)",
+  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND archived = false)",
   "describe": {
     "columns": [
       {
@@ -22,6 +22,11 @@
         "ordinal": 3,
         "name": "concurrency_time_window_s",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
       }
     ],
     "parameters": {
@@ -34,8 +39,9 @@
       false,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "9289709b2727425be0e7f1e341d6acd46e1e79d5018f6a18cb41d50ca3f4ba11"
+  "hash": "cf21b61aa5b5e47744235c860e2d9b90d7654a4b5d0ec32085b178399f701bfc"
 }

backend/.sqlx/query-dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -95,6 +125,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-e25829ed40f35a0f59a9838d030a7f888ca88441e069e73a52d75450aecae70e.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET canceled = true,  canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE workspace_id = $1 AND schedule_path IS NULL RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e25829ed40f35a0f59a9838d030a7f888ca88441e069e73a52d75450aecae70e"
+}

backend/.sqlx/query-ebac391c83dae67831502fbe3f1034210afce7902a69f7c42bb68160e0464d71.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select path, tag, concurrent_limit, concurrency_time_window_s from script where hash = $1 AND workspace_id = $2",
+  "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -22,6 +22,11 @@
         "ordinal": 3,
         "name": "concurrency_time_window_s",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
       }
     ],
     "parameters": {
@@ -34,8 +39,9 @@
       false,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "d164caec6637ab53d8c2e0faad653a458f31a976bb23c3a43d35b8476b066954"
+  "hash": "ebac391c83dae67831502fbe3f1034210afce7902a69f7c42bb68160e0464d71"
 }

backend/.sqlx/query-eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -95,6 +125,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json

@@ -38,7 +38,8 @@
                 "http",
                 "graphql",
                 "postgresql",
-                "noop"
+                "noop",
+                "appdependencies"
               ]
             }
           }
@@ -59,7 +60,8 @@
                 "http",
                 "graphql",
                 "postgresql",
-                "noop"
+                "noop",
+                "appdependencies"
               ]
             }
           }

backend/.sqlx/query-fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480.json

@@ -72,6 +72,36 @@
         "ordinal": 13,
         "name": "on_failure",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
       }
     ],
     "parameters": {
@@ -93,6 +123,12 @@
       false,
       true,
       false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
       true
     ]
   },

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.151.2"
+version = "1.167.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.151.2"
+version = "1.167.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -36,7 +36,8 @@ incremental = true
 
 [features]
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
-benchmark = ["windmill-api/benchmark"]
+benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
+flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 
 [dependencies]
 anyhow.workspace = true
@@ -131,17 +132,18 @@ tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
-deno_fetch = "0.133.0"
-deno_console = "0.109.0"
-deno_url = "0.109.0"
-deno_webidl = "0.109.0"
-deno_web = "0.140.0"
-deno_core = "0.191.0"
-deno_ast = { version = "0.27.2", features = ["transpiling"] }
+deno_fetch = "0.139.0"
+deno_console = "0.115.0"
+deno_url = "0.115.0"
+deno_webidl = "0.115.0"
+deno_web = "0.146.0"
+deno_core = "0.200.0"
+deno_ast = { version = "0.28.0", features = ["transpiling"] }
 async-recursion = "^1"
-swc_common = "0.31.12"
-swc_ecma_parser = "0.136.2"
-swc_ecma_ast = "0.106.2"
+swc_common = "0.31.21"
+swc_ecma_parser = "0.137.15"
+swc_ecma_ast = "0.107.7"
+swc_ecma_visit = "0.93.7"
 base64 = "0.21.0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
@@ -187,4 +189,4 @@ samael = { version = "0.0.12", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
 rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
-pem = "3.0.1"
+pem = "3.0.1"

backend/migrations/20230818085630_add_schedule_recovery.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE schedule DROP COLUMN on_recovery;

backend/migrations/20230818085630_add_schedule_recovery.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN on_recovery VARCHAR(1000);

backend/migrations/20230820161609_add_schedule_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230820161609_add_schedule_index.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS scheduled_root_job ON completed_job (workspace_id, schedule_path, started_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index_by_path ON completed_job (workspace_id, script_path, job_kind, created_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index ON completed_job (workspace_id, job_kind, created_at) WHERE parent_job is NULL;
+DROP INDEX IF EXISTS index_completed_on_script_hash;
+DROP INDEX IF EXISTS index_completed_on_script_path;
+DROP INDEX IF EXISTS index_completed_on_schedule_path;
+DROP INDEX IF EXISTS index_completed_on_workspace_id;
+DROP INDEX IF EXISTS index_completed_on_created_at;
+
+DROP INDEX IF EXISTS index_queue_on_script_path;
+DROP INDEX IF EXISTS index_queue_on_script_hash;
+DROP INDEX IF EXISTS index_queue_on_workspace_id;
+DROP INDEX IF EXISTS index_queue_on_scheduled_for;
+DROP INDEX IF EXISTS index_queue_on_tag;
+DROP INDEX IF EXISTS index_queue_on_running;
+DROP INDEX IF EXISTS index_queue_on_created;
+
+
+CREATE INDEX IF NOT EXISTS root_queue_index_by_path ON queue (workspace_id, created_at);
+CREATE INDEX IF NOT EXISTS root_queue_index ON queue (job_kind, tag, scheduled_for, created_at) WHERE running is false;
+CREATE INDEX IF NOT EXISTS root_queue_index_suspended ON queue (job_kind, tag, suspend_until, suspend, scheduled_for, created_at) WHERE suspend_until is not null;
+
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_queue ON queue (workspace_id, script_path, started_at) WHERE concurrent_limit is not null;
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_completed_job ON completed_job (workspace_id, script_path, started_at);

backend/migrations/20230823074735_schedule_handlers_times.down.sql

@@ -0,0 +1,7 @@
+-- Add down migration script here
+ALTER TABLE schedule 
+DROP COLUMN on_failure_times,
+DROP COLUMN on_failure_exact,
+DROP COLUMN on_failure_extra_args,
+DROP COLUMN on_recovery_times,
+DROP COLUMN on_recovery_extra_args;

backend/migrations/20230823074735_schedule_handlers_times.up.sql

@@ -0,0 +1,7 @@
+-- Add up migration script here
+ALTER TABLE schedule
+ADD COLUMN on_failure_times INTEGER,
+ADD COLUMN on_failure_exact BOOLEAN,
+ADD COLUMN on_failure_extra_args json,
+ADD COLUMN on_recovery_times INTEGER,
+ADD COLUMN on_recovery_extra_args json;

backend/migrations/20230826001320_appdependencies.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230826001320_appdependencies.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appdependencies';

backend/migrations/20230828202124_add_cache_ttl.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230828202124_add_cache_ttl.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN cache_ttl INTEGER;
+ALTER TABLE queue ADD COLUMN cache_ttl INTEGER;

backend/migrations/20230829202651_add_global_settings.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230829202651_add_global_settings.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+CREATE TABLE global_settings (
+    name VARCHAR(255) PRIMARY KEY,
+    value JSONB NOT NULL,
+    updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+);

backend/migrations/20230831210028_add_q_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230831210028_add_q_index.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS queue_sort ON queue (scheduled_for, created_at) WHERE running = false;

backend/migrations/20230902192640_add_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230902192640_add_index.up.sql

@@ -0,0 +1,13 @@
+-- Add up migration script here
+DROP INDEX IF EXISTS queue_sort;
+CREATE INDEX IF NOT EXISTS queue_sort ON queue (scheduled_for, created_at, tag) WHERE running = false;
+
+CREATE TABLE IF NOT EXISTS concurrency_counter (
+  concurrency_id VARCHAR(1000) PRIMARY KEY,
+  counter INTEGER NOT NULL
+);
+
+DROP INDEX IF EXISTS root_queue_index_suspended;
+
+DROP INDEX IF EXISTS queue_suspended;
+CREATE INDEX IF NOT EXISTS queue_suspended ON queue (created_at, suspend_until, suspend, tag) WHERE suspend_until is not null;

backend/out.json

@@ -0,0 +1 @@
+{"completed":true,"result":{"error":{"name":"Exception","stack":"  File \"/tmp/u/ruben/gentle_script.py\", line 35, in main\n    print(wmill.get_variable(\"u/admin/adorable_postgresql\"))\n          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n  File \"/tmp/windmill/cache/pip/wmill==1.166.0/wmill/client.py\", line 336, in get_variable\n    raise Exception(\n","message":"Variable at path u/admin/adorable_postgresql does not exist or you do not have read permissions on it"}}}

backend/parsers/windmill-parser-go/Cargo.toml

@@ -12,4 +12,6 @@ path = "./src/lib.rs"
 windmill-parser.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
-gosyn.workspace = true
+gosyn.workspace = true
+lazy_static.workspace = true
+regex.workspace = true

backend/parsers/windmill-parser-go/src/lib.rs

@@ -6,8 +6,13 @@ use gosyn::{
 };
 use itertools::Itertools;
 
+use regex::Regex;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
 
+lazy_static::lazy_static! {
+    pub static ref REQUIRE_PARSE: Regex = Regex::new(r"//require (.*)\n").unwrap();
+}
+
 pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let filtered_code = filter_non_main(code);
     let file = parse_source(&filtered_code).map_err(|x| anyhow::anyhow!(x.to_string()))?;
@@ -46,7 +51,14 @@ pub fn parse_go_imports(code: &str) -> anyhow::Result<Vec<String>> {
         })
         .collect();
     imports.sort();
-    Ok(imports)
+    Ok([
+        imports,
+        REQUIRE_PARSE
+            .captures_iter(code)
+            .map(|x| x[1].to_string())
+            .collect_vec(),
+    ]
+    .concat())
 }
 
 fn get_name(param: &Field) -> String {

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -32,6 +32,9 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "smb" => "pysmb",
     "PIL" => "Pillow",
     "googleapiclient" => "google-api-python-client",
+    "dateutil" => "python-dateutil",
+    "mailparser" => "mail-parser",
+    "mailparser-reply" => "mail-parser-reply",
 };
 
 fn replace_import(x: String) -> String {
@@ -145,19 +148,20 @@ pub async fn parse_python_imports(
             .collect();
         for n in nimports.iter() {
             let nested = if n.starts_with("relative:") {
+                let rpath = n.replace("relative:", "");
                 let code = sqlx::query_scalar!(
                     r#"
                     SELECT content FROM script WHERE path = $1 AND workspace_id = $2
                     AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND 
                     workspace_id = $2)
                     "#,
-                    n.replace("relative:", ""),
+                    &rpath,
                     w_id
                 )
                 .fetch_optional(db)
                 .await?
                 .unwrap_or_else(|| "".to_string());
-                parse_python_imports(&code, w_id, path, db).await?
+                parse_python_imports(&code, w_id, &rpath, db).await?
             } else {
                 vec![replace_import(n.to_string())]
             };

backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql

@@ -2556,4 +2556,27 @@ import numpy as np
 '{}',
 '',
 '',
-'f/foo/baz', -28028598712388161, 'python3', '');
+'f/foo/baz', -28028598712388161, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import difffolder
+from .bar import main
+',
+'{}',
+'',
+'',
+'f/foobar/baz', -28028598712388160, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import innerdifffolder
+',
+'{}',
+'',
+'',
+'f/foobar/bar', -28028598712388159, 'python3', '');

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -57,7 +57,7 @@ from f.foo.bar import main1
 from .baz import main2
 from ..foo.bar import main3
 import f.foo.bar as bar
-
+from ..foobar.baz import main4
 
 def main():
     pass
@@ -65,7 +65,17 @@ def main():
 ";
         let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
         println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, ["numpy", "pandas", "pandas2", "requests"]);
+        assert_eq!(
+            r,
+            [
+                "difffolder",
+                "innerdifffolder",
+                "numpy",
+                "pandas",
+                "pandas2",
+                "requests"
+            ]
+        );
 
         Ok(())
     }

backend/parsers/windmill-parser-ts/Cargo.toml

@@ -18,6 +18,7 @@ windmill-parser.workspace = true
 swc_common.workspace = true
 swc_ecma_parser.workspace = true
 swc_ecma_ast.workspace = true
+swc_ecma_visit.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
 convert_case.workspace = true

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -9,21 +9,78 @@ use regex::Regex;
  */
 // use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
 use serde_json::Value;
+use std::collections::HashSet;
+use swc_ecma_visit::{noop_visit_type, Visit, VisitWith};
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
 
 use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
 use swc_ecma_ast::{
     ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
-    ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str, TsArrayType,
-    TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType,
+    MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str,
+    TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType,
     TsParenthesizedType, TsPropertySignature, TsType, TsTypeAnn, TsTypeElement, TsTypeLit,
     TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
 };
-use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+use swc_ecma_parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax, TsConfig};
 
 #[cfg(target_arch = "wasm32")]
 use wasm_bindgen::prelude::*;
 
+struct OutputFinder {
+    idents: HashSet<(String, String)>,
+}
+
+impl Visit for OutputFinder {
+    noop_visit_type!();
+
+    fn visit_member_expr(&mut self, m: &MemberExpr) {
+        m.obj.visit_with(self);
+        if let MemberProp::Computed(c) = &m.prop {
+            c.visit_with(self);
+        }
+        match m {
+            MemberExpr { obj, prop: MemberProp::Ident(Ident { sym, .. }), .. } => {
+                match *obj.to_owned() {
+                    Expr::Ident(Ident { sym: sym_i, .. }) => {
+                        self.idents.insert((sym_i.to_string(), sym.to_string()));
+                    }
+                    _ => (),
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Es(EsConfig { jsx: false, ..Default::default() }),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let expr = parser
+        .parse_module()
+        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?;
+
+    let mut visitor = OutputFinder { idents: HashSet::new() };
+    swc_ecma_visit::visit_module(&mut visitor, &expr);
+
+    Ok(visitor.idents.into_iter().collect())
+}
+
 pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainArgSignature> {
     let cm: Lrc<SourceMap> = Default::default();
     let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
@@ -308,8 +365,8 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 extern "C" {
     pub fn eval(s: &str) -> JsValue;
     pub fn alert(s: &str);
-    // #[wasm_bindgen(js_namespace = console)]
-    // fn log(s: &str);
+    #[wasm_bindgen(js_namespace = console)]
+    fn log(s: &str);
 
 }
 

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.147.3",
+  "version": "1.154.3",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts

@@ -9,6 +9,11 @@ export function parse_deno(code: string): string;
 * @param {string} code
 * @returns {string}
 */
+export function parse_outputs(code: string): string;
+/**
+* @param {string} code
+* @returns {string}
+*/
 export function parse_bash(code: string): string;
 /**
 * @param {string} code
@@ -56,6 +61,7 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
 export interface InitOutput {
   readonly memory: WebAssembly.Memory;
   readonly parse_deno: (a: number, b: number, c: number) => void;
+  readonly parse_outputs: (a: number, b: number, c: number) => void;
   readonly parse_bash: (a: number, b: number, c: number) => void;
   readonly parse_powershell: (a: number, b: number, c: number) => void;
   readonly parse_go: (a: number, b: number, c: number) => void;

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -220,6 +220,29 @@ export function parse_deno(code) {
     }
 }
 
+/**
+* @param {string} code
+* @returns {string}
+*/
+export function parse_outputs(code) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.parse_outputs(retptr, ptr0, len0);
+        var r0 = getInt32Memory0()[retptr / 4 + 0];
+        var r1 = getInt32Memory0()[retptr / 4 + 1];
+        deferred2_0 = r0;
+        deferred2_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
+
 /**
 * @param {string} code
 * @returns {string}
@@ -469,6 +492,10 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
+    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
@@ -520,10 +547,6 @@ function __wbg_get_imports() {
         const ret = new Error(getStringFromWasm0(arg0, arg1));
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_eval_2824c9167fe66e96 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts

@@ -2,6 +2,7 @@
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
 export function parse_deno(a: number, b: number, c: number): void;
+export function parse_outputs(a: number, b: number, c: number): void;
 export function parse_bash(a: number, b: number, c: number): void;
 export function parse_powershell(a: number, b: number, c: number): void;
 export function parse_go(a: number, b: number, c: number): void;

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -1,6 +1,7 @@
-use serde_json;
+use serde_json::{self, json};
 use wasm_bindgen::prelude::*;
 use windmill_parser::MainArgSignature;
+use windmill_parser_ts::parse_expr_for_ids;
 
 fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
     if let Ok(r) = r {
@@ -15,6 +16,17 @@ pub fn parse_deno(code: &str) -> String {
     wrap_sig(windmill_parser_ts::parse_deno_signature(code, false))
 }
 
+#[wasm_bindgen]
+pub fn parse_outputs(code: &str) -> String {
+    let parsed = parse_expr_for_ids(code);
+    let r = if let Ok(parsed) = parsed {
+        json!({ "outputs": parsed })
+    } else {
+        json!({"error": parsed.err().unwrap().to_string()})
+    };
+    return serde_json::to_string(&r).unwrap();
+}
+
 #[wasm_bindgen]
 pub fn parse_bash(code: &str) -> String {
     wrap_sig(windmill_parser_bash::parse_bash_sig(code))

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -1,7 +1,7 @@
 use serde_json::json;
 use wasm_bindgen_test::wasm_bindgen_test;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
-use windmill_parser_ts::parse_deno_signature;
+use windmill_parser_ts::{parse_deno_signature, parse_expr_for_ids};
 
 #[wasm_bindgen_test]
 fn test_parse_deno_sig() -> anyhow::Result<()> {
@@ -256,3 +256,22 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
 
     Ok(())
 }
+
+#[wasm_bindgen_test]
+fn test_parse_extract_ident() -> anyhow::Result<()> {
+    let code = "
+    let foo = 3;
+    bar
+    baroof.foob.ar
+    foobar[barfoo.x]
+";
+    assert_eq!(
+        parse_expr_for_ids(code)?,
+        vec![
+            ("baroof".to_string(), "foob".to_string()),
+            ("barfoo".to_string(), "x".to_string())
+        ]
+    );
+
+    Ok(())
+}

backend/src/main.rs

@@ -20,7 +20,7 @@ use tokio::{
     sync::RwLock,
 };
 use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{utils::rd_string, METRICS_ADDR};
+use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -40,15 +40,19 @@ mod monitor;
 async fn main() -> anyhow::Result<()> {
     dotenv::dotenv().ok();
 
+    #[cfg(not(feature = "flamegraph"))]
     windmill_common::tracing_init::initialize_tracing();
 
+    #[cfg(feature = "flamegraph")]
+    let _guard = windmill_common::tracing_init::setup_flamegraph();
+
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(DEFAULT_NUM_WORKERS as i32);
 
     if num_workers > 1 {
-        tracing::warn!("We recommend using at most 1 worker per container, use more only if you know what you are doing.");
+        tracing::warn!("We STRONGLY recommend using at most 1 worker per container, unless this worker is dedicated to native jobs only. ");
     }
     let metrics_addr: Option<SocketAddr> = *METRICS_ADDR;
 
@@ -92,6 +96,7 @@ async fn main() -> anyhow::Result<()> {
     tracing::info!("Database connected");
 
     let rsmq = if let Some(config) = rsmq_config {
+        tracing::info!("Redis config set: {:?}", config);
         Some(rsmq_async::MultiplexedRsmq::new(config).await.unwrap())
     } else {
         None
@@ -119,68 +124,7 @@ Windmill Community Edition {GIT_VERSION}
 ##############################"
     );
 
-    display_config(vec![
-        "DISABLE_NSJAIL",
-        "DISABLE_SERVER",
-        "NUM_WORKERS",
-        "METRICS_ADDR",
-        "JSON_FMT",
-        "BASE_URL",
-        "TIMEOUT",
-        "ZOMBIE_JOB_TIMEOUT",
-        "RESTART_ZOMBIE_JOBS",
-        "SLEEP_QUEUE",
-        "MAX_LOG_SIZE",
-        "SERVER_BIND_ADDR",
-        "PORT",
-        "KEEP_JOB_DIR",
-        "S3_CACHE_BUCKET",
-        "TAR_CACHE_RATE",
-        "COOKIE_DOMAIN",
-        "PYTHON_PATH",
-        "DENO_PATH",
-        "GO_PATH",
-        "GOPRIVATE",
-        "GOPROXY",
-        "NETRC",
-        "PIP_INDEX_URL",
-        "PIP_EXTRA_INDEX_URL",
-        "PIP_TRUSTED_HOST",
-        "PATH",
-        "HOME",
-        "DATABASE_CONNECTIONS",
-        "TIMEOUT_WAIT_RESULT",
-        "QUEUE_LIMIT_WAIT_RESULT",
-        "DENO_AUTH_TOKENS",
-        "DENO_FLAGS",
-        "NPM_CONFIG_REGISTRY",
-        "PIP_LOCAL_DEPENDENCIES",
-        "ADDITIONAL_PYTHON_PATHS",
-        "INCLUDE_HEADERS",
-        "INSTANCE_EVENTS_WEBHOOK",
-        "CLOUD_HOSTED",
-        "GLOBAL_CACHE_INTERVAL",
-        "WORKER_TAGS",
-        "CUSTOM_TAGS",
-        "JOB_RETENTION_SECS",
-        "WAIT_RESULT_FAST_POLL_DURATION_SECS",
-        "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
-        "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
-        "EXIT_AFTER_NO_JOB_FOR_SECS",
-        "REQUEST_SIZE_LIMIT",
-        "SMTP_HOST",
-        "SMTP_USERNAME",
-        "SMTP_PORT",
-        "SMTP_TLS_IMPLICIT",
-        "CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
-        "GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
-    ]);
-
-    if std::env::var("WHITELIST_WORKSPACES").is_ok()
-        || std::env::var("BLACKLIST_WORKSPACES").is_ok()
-    {
-        panic!("WHITELIST_WORKSPACES and BLACKLIST_WORKSPACES have been removed, please use Worker Groups instead");
-    }
+    display_config(&ENV_SETTINGS);
 
     tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
     if let Some(ref smtp) = *SMTP_CLIENT {
@@ -244,8 +188,12 @@ Windmill Community Edition {GIT_VERSION}
 
         let metrics_f = async {
             match metrics_addr {
-                Some(addr) => {
-                    windmill_common::serve_metrics(addr, rx.resubscribe(), num_workers > 0)
+                Some(_addr) => {
+                    #[cfg(not(feature = "enterprise"))]
+                    panic!("Metrics are only available in the Enterprise Edition");
+
+                    #[cfg(feature = "enterprise")]
+                    windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
                         .await
                         .map_err(anyhow::Error::from)
                 }
@@ -260,7 +208,7 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
-fn display_config(envs: Vec<&str>) {
+fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
         envs.iter()

backend/src/monitor.rs

@@ -10,7 +10,7 @@ use windmill_common::{
     METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
 };
 
 lazy_static::lazy_static! {
@@ -116,7 +116,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             &job.workspace_id,
             &job.permissioned_as,
             "ephemeral-zombie-jobs",
-            *SESSION_TOKEN_EXPIRY,
+            *SCRIPT_TOKEN_EXPIRY,
             &job.email,
         )
         .await
@@ -133,7 +133,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         let _ = handle_job_error(
             db,
             &client,
-            job,
+            &job,
             error::Error::ExecutionErr(format!(
                 "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
                 last_ping
@@ -145,7 +145,6 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             true,
             same_worker_tx_never_used,
             "",
-            base_internal_url,
             rsmq.clone(),
         )
         .await;

backend/tests/fixtures/base.sql

@@ -11,8 +11,65 @@ INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
 INSERT INTO workspace_key(workspace_id, kind, key) VALUES
 	('test-workspace', 'cloud', 'test-key');
 
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('test-workspace');
+
 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);
+
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 

backend/tests/worker.rs

@@ -1,19 +1,26 @@
 use std::sync::Arc;
 
+use chrono::Timelike;
+use futures::StreamExt;
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::sync::RwLock;
+use tokio::{
+    sync::RwLock,
+    time::{timeout, Duration},
+};
 use windmill_api::jobs::{CompletedJob, Job};
-use windmill_api_client::types::{CreateFlowBody, RawScript};
+use windmill_api_client::types::{
+    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+};
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
     jobs::{JobPayload, RawCode},
     scripts::ScriptLang,
 };
-use windmill_queue::get_queued_job;
+use windmill_queue::{get_queued_job, PushIsolationLevel};
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -839,8 +846,10 @@ impl RunJob {
 
     async fn push(self, db: &Pool<Postgres>) -> Uuid {
         let RunJob { payload, args } = self;
+        let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None);
         let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
-            (None, db.begin().await.unwrap()).into(),
+            &db,
+            tx,
             "test-workspace",
             payload,
             args,
@@ -1046,6 +1055,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                         iterator: InputTransform::Javascript { expr: "result".to_string() },
                         skip_failures: false,
                         parallel: false,
+                        parallelism: None,
                         modules: vec![FlowModule {
                             id: "c".to_string(),
                             value: FlowModuleValue::RawScript {
@@ -1189,6 +1199,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         iterator: InputTransform::Static { value: json!([1, 2, 3]) },
                         skip_failures: false,
                         parallel: false,
+                        parallelism: None,
                         modules: vec![
                             FlowModule {
                                 id: "d".to_string(),
@@ -1608,6 +1619,7 @@ func main(derp string) (string, error) {
         language: ScriptLang::Go,
         concurrent_limit: None,
         concurrency_time_window_s: None,
+        cache_ttl: None
     }))
     .arg("derp", json!("world"))
     .run_until_complete(&db, port)
@@ -1637,6 +1649,7 @@ echo "hello $msg"
         language: ScriptLang::Bash,
         concurrent_limit: None,
         concurrency_time_window_s: None,
+        cache_ttl: None
     }))
     .arg("msg", json!("world"))
     .run_until_complete(&db, port)
@@ -1664,6 +1677,7 @@ def main():
         lock: None,
         concurrent_limit: None,
         concurrency_time_window_s: None,
+        cache_ttl: None
     });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -1696,6 +1710,7 @@ def main():
         lock: None,
         concurrent_limit: None,
         concurrency_time_window_s: None,
+        cache_ttl: None
     });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -1727,6 +1742,7 @@ def main():
         lock: None,
         concurrent_limit: None,
         concurrency_time_window_s: None,
+        cache_ttl: None
     });
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -2540,14 +2556,14 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
             assert!(matches!(
                 m.value,
                 windmill_api_client::types::FlowModuleValue::RawScript(RawScript {
-                    language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
+                    language: windmill_api_client::types::RawScriptLanguage::Bash,
                     lock: Some(ref lock),
                     ..
                 }) if lock == "")
                 || matches!(
                 m.value,
                 windmill_api_client::types::FlowModuleValue::RawScript(RawScript{
-                    language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
+                    language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3 | windmill_api_client::types::RawScriptLanguage::Deno,
                     lock: Some(ref lock),
                     ..
                 }) if lock.len() > 0)
@@ -2569,3 +2585,274 @@ async fn test_rust_client(db: Pool<Postgres>) {
     .await
     .unwrap();
 }
+
+#[sqlx::test(fixtures("base"))]
+async fn test_script_schedule_handlers(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(true));
+
+    let now = chrono::Utc::now();
+    // add 5 seconds to now
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+
+    let schedule = NewSchedule {
+        args: ScriptArgs::from(args),
+        enabled: Some(true),
+        is_flow: false,
+        on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+        on_failure_times: None,
+        on_failure_exact: None,
+        on_failure_extra_args: None,
+        on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+        on_recovery_times: None,
+        on_recovery_extra_args: None,
+        path: "f/system/failing_script_schedule".to_string(),
+        script_path: "f/system/failing_script".to_string(),
+        timezone: "UTC".to_string(),
+        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+    };
+
+    let _ = client.create_schedule("test-workspace", &schedule).await;
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed error job
+
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // error handler
+
+            if uuid.is_err() {
+                panic!("schedule error handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path != Some("f/system/schedule_error_handler".to_string())
+            {
+                panic!(
+                    "a script was run after main job execution but was not schedule error handler"
+                );
+            }
+        },
+        port,
+    )
+    .await;
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(false));
+    let now = chrono::Utc::now();
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+    client
+        .update_schedule(
+            "test-workspace",
+            "f/system/failing_script_schedule",
+            &EditSchedule {
+                args: ScriptArgs::from(args),
+                on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+                on_failure_times: None,
+                on_failure_exact: None,
+                on_failure_extra_args: None,
+                on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+                on_recovery_times: None,
+                on_recovery_extra_args: None,
+                timezone: "UTC".to_string(),
+                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+            },
+        )
+        .await
+        .unwrap();
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed working job
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // recovery handler
+
+            if uuid.is_err() {
+                panic!("schedule recovery handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+            
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path
+                    != Some("f/system/schedule_recovery_handler".to_string())
+            {
+                panic!("a script was run after main job execution but was not schedule recovery handler");
+            }
+        },
+        port,
+    )
+    .await;
+}
+
+
+#[sqlx::test(fixtures("base"))]
+async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(true));
+
+    let now = chrono::Utc::now();
+    // add 5 seconds to now
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+
+    let schedule = NewSchedule {
+        args: ScriptArgs::from(args),
+        enabled: Some(true),
+        is_flow: true,
+        on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+        on_failure_times: None,
+        on_failure_exact: None,
+        on_failure_extra_args: None,
+        on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+        on_recovery_times: None,
+        on_recovery_extra_args: None,
+        path: "f/system/failing_flow_schedule".to_string(),
+        script_path: "f/system/failing_flow".to_string(),
+        timezone: "UTC".to_string(),
+        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+    };
+
+    let _ = client.create_schedule("test-workspace", &schedule).await;
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed error step
+            str.next().await; // completed error flow
+
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // error handler
+
+            if uuid.is_err() {
+                panic!("schedule error handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path != Some("f/system/schedule_error_handler".to_string())
+            {
+                panic!(
+                    "a script was run after main job execution but was not schedule error handler"
+                );
+            }
+        },
+        port,
+    )
+    .await;
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(false));
+    let now = chrono::Utc::now();
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+    client
+        .update_schedule(
+            "test-workspace",
+            "f/system/failing_flow_schedule",
+            &EditSchedule {
+                args: ScriptArgs::from(args),
+                on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+                on_failure_times: None,
+                on_failure_exact: None,
+                on_failure_extra_args: None,
+                on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+                on_recovery_times: None,
+                on_recovery_extra_args: None,
+                timezone: "UTC".to_string(),
+                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+            },
+        )
+        .await
+        .unwrap();
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed working step
+            str.next().await; // completed working flow
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // recovery handler
+
+            if uuid.is_err() {
+                panic!("schedule recovery handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+            
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path
+                    != Some("f/system/schedule_recovery_handler".to_string())
+            {
+                panic!("a script was run after main job execution but was not schedule recovery handler");
+            }
+        },
+        port,
+    )
+    .await;
+}

backend/windmill-api-client/Cargo.toml

@@ -19,9 +19,11 @@ uuid.workspace = true
 serde_json.workspace = true
 rand.workspace = true
 base64.workspace = true
+openapiv3 = "=1.0.2"
 
 [build-dependencies]
 prettyplease = "0.1.25"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 serde_json = "1.0"
-syn = "1.0"
+syn = "1.0"
+openapiv3 = "=1.0.2"

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.151.2
+  version: 1.167.0
   title: Windmill API
 
   contact:
@@ -522,6 +522,59 @@ paths:
               schema:
                 type: boolean
 
+  /settings/global/{key}:
+    get:
+      summary: get global settings
+      operationId: getGlobal
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/Key"
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
+    post:
+      summary: post global settings
+      operationId: setGlobal
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/Key"
+      requestBody:
+        description: value set
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                value: {}
+
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /settings/local:
+    get:
+      summary: get local settings
+      operationId: getLocal
+      tags:
+        - setting
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -3597,6 +3650,8 @@ paths:
                       type: string
                     path:
                       type: string
+                    cache_ttl:
+                      type: integer
                   required:
                     - content
                     - language
@@ -3811,6 +3866,45 @@ paths:
                 items:
                   $ref: "#/components/schemas/QueuedJob"
 
+  /w/{workspace}/jobs/queue/count:
+    get:
+      summary: get queue count
+      operationId: getQueueCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: queue count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
+  /w/{workspace}/jobs/queue/cancel_all:
+    post:
+      summary: cancel all jobs
+      operationId: cancelAll
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: uuids of canceled jobs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
   /w/{workspace}/jobs/completed/list:
     get:
       summary: list all available completed jobs
@@ -3869,6 +3963,8 @@ paths:
         - $ref: "#/components/parameters/ScriptExactHash"
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
+        - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/CreatedOrStartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/Tag"
@@ -3898,6 +3994,21 @@ paths:
                 items:
                   $ref: "#/components/schemas/Job"
 
+  /jobs/db_clock:
+    get:
+      summary: get db clock
+      operationId: getDbClock
+      tags:
+        - job
+      responses:
+        "200":
+          description:
+            the timestamp of the db that can be used to compute the drift
+          content:
+            application/json:
+              schema:
+                type: integer
+
   /w/{workspace}/jobs_u/get/{id}:
     get:
       summary: get job
@@ -5392,6 +5503,12 @@ components:
       name: token
 
   parameters:
+    Key:
+      name: key
+      in: path
+      required: true
+      schema:
+        type: string
     WorkspaceId:
       name: workspace
       in: path
@@ -5552,14 +5669,32 @@ components:
         type: string
     StartedBefore:
       name: started_before
-      description: filter on created before (inclusive) timestamp
+      description: filter on started before (inclusive) timestamp
       in: query
       schema:
         type: string
         format: date-time
     StartedAfter:
       name: started_after
-      description: filter on created after (exclusive) timestamp
+      description: filter on started after (exclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
+    CreatedOrStartedAfter:
+      name: created_or_started_after
+      description:
+        filter on created_at for non non started job and started_at otherwise
+        after (exclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
+    CreatedOrStartedBefore:
+      name: created_or_started_before
+      description:
+        filter on created_at for non non started job and started_at otherwise
+        before (inclusive) timestamp
       in: query
       schema:
         type: string
@@ -5757,7 +5892,8 @@ components:
           type: array
           items:
             type: integer
-
+        cache_ttl:
+          type: number
       required:
         - hash
         - path
@@ -5831,6 +5967,8 @@ components:
           type: array
           items:
             type: integer
+        cache_ttl:
+          type: number
       required:
         - path
         - summary
@@ -5960,6 +6098,8 @@ components:
               "script",
               "preview",
               "dependencies",
+              "flowdependencies",
+              "appdependencies",
               "flow",
               "flowpreview",
               "script_hub",
@@ -6064,6 +6204,8 @@ components:
               "preview",
               "dependencies",
               "flow",
+              "flowdependencies",
+              "appdependencies",
               "flowpreview",
               "script_hub",
               "identity",
@@ -6754,6 +6896,18 @@ components:
         on_failure:
           # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
           type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
       required:
         - path
         - edited_by
@@ -6807,6 +6961,18 @@ components:
         on_failure:
           # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
           type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
       required:
         - path
         - schedule
@@ -6827,6 +6993,18 @@ components:
         on_failure:
           # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
           type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
       required:
         - schedule
         - timezone

backend/windmill-api/src/apps.rs

@@ -8,8 +8,8 @@ use std::collections::HashMap;
  * LICENSE-AGPL for a copy of the license.
  */
 use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed, OptAuthed},
+    db::{ApiAuthed, DB},
+    users::{require_owner_of_path, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
@@ -30,6 +30,7 @@ use std::str;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     apps::ListAppQuery,
+    db::UserDB,
     error::{to_anyhow, Error, JsonResult, Result},
     jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
     users::username_to_permissioned_as,
@@ -37,7 +38,7 @@ use windmill_common::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, QueueTransaction};
+use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -159,7 +160,7 @@ pub struct EditApp {
 }
 
 async fn list_apps(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -220,7 +221,7 @@ async fn list_apps(
 }
 
 async fn get_app(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<AppWithLastVersion> {
@@ -245,7 +246,7 @@ async fn get_app(
 }
 
 async fn get_app_w_draft(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<AppWithLastVersionAndDraft> {
@@ -262,7 +263,7 @@ async fn get_app_w_draft(
         INNER JOIN app_version ON
         app_version.id = app.versions[array_upper(app.versions, 1)]
         LEFT JOIN draft ON 
-        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' 
+        app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' 
         WHERE app.path = $1 AND app.workspace_id = $2"#,
         path.to_owned(),
         &w_id
@@ -276,7 +277,7 @@ async fn get_app_w_draft(
 }
 
 async fn get_app_by_id(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, id)): Path<(String, i64)>,
 ) -> JsonResult<AppWithLastVersion> {
@@ -341,7 +342,7 @@ async fn get_public_app_by_secret(
 }
 
 async fn get_secret_id(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
@@ -369,16 +370,18 @@ async fn get_secret_id(
 }
 
 async fn create_app(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(webhook): Extension<WebhookShared>,
     Path(w_id): Path<String>,
     Json(mut app): Json<CreateApp>,
 ) -> Result<(StatusCode, String)> {
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
 
     app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
-    app.policy.on_behalf_of_email = Some(authed.email);
+    app.policy.on_behalf_of_email = Some(authed.email.clone());
 
     if &app.path == "" {
         return Err(Error::BadRequest("App path cannot be empty".to_string()));
@@ -389,7 +392,7 @@ async fn create_app(
         &app.path,
         w_id
     )
-    .fetch_one(&mut *tx)
+    .fetch_one(&mut tx)
     .await?
     .unwrap_or(false);
 
@@ -405,7 +408,7 @@ async fn create_app(
         &app.path,
         &w_id
     )
-    .execute(&mut *tx)
+    .execute(&mut tx)
     .await?;
 
     let id = sqlx::query_scalar!(
@@ -418,7 +421,7 @@ async fn create_app(
         json!(app.policy),
         app.draft_only,
     )
-    .fetch_one(&mut *tx)
+    .fetch_one(&mut tx)
     .await?;
 
     let v_id = sqlx::query_scalar!(
@@ -430,7 +433,7 @@ async fn create_app(
         serde_json::to_string(&app.value).unwrap(),
         authed.username,
     )
-    .fetch_one(&mut *tx)
+    .fetch_one(&mut tx)
     .await?;
 
     sqlx::query!(
@@ -438,11 +441,11 @@ async fn create_app(
         v_id,
         id
     )
-    .execute(&mut *tx)
+    .execute(&mut tx)
     .await?;
 
     audit_log(
-        &mut *tx,
+        &mut tx,
         &authed.username,
         "apps.create",
         ActionKind::Create,
@@ -452,6 +455,31 @@ async fn create_app(
     )
     .await?;
 
+    let tx = PushIsolationLevel::Transaction(tx);
+    let (dependency_job_uuid, tx) = push(
+        &db,
+        tx,
+        &w_id,
+        JobPayload::AppDependencies { path: app.path.clone(), version: v_id },
+        serde_json::Map::new(),
+        &authed.username,
+        &authed.email,
+        windmill_common::users::username_to_permissioned_as(&authed.username),
+        None,
+        None,
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+        None,
+        None,
+        None,
+    )
+    .await?;
+    tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
     tx.commit().await?;
     webhook.send_message(
         w_id.clone(),
@@ -461,7 +489,7 @@ async fn create_app(
     Ok((StatusCode::CREATED, app.path))
 }
 
-async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
+async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchUiData?approved=true",
@@ -472,7 +500,7 @@ async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::V
 }
 
 pub async fn get_hub_app_by_id(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
     Path(id): Path<i32>,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
@@ -489,7 +517,7 @@ pub async fn get_hub_app_by_id(
 }
 
 async fn delete_app(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -539,9 +567,11 @@ async fn delete_app(
 }
 
 async fn update_app(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
 ) -> Result<String> {
@@ -549,7 +579,7 @@ async fn update_app(
 
     let path = path.to_path();
 
-    let mut tx = user_db.begin(&authed).await?;
+    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
 
     let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() {
         let mut sqlb = SqlBuilder::update_table("app");
@@ -566,7 +596,7 @@ async fn update_app(
                     npath,
                     w_id
                 )
-                .fetch_one(&mut *tx)
+                .fetch_one(&mut tx)
                 .await?
                 .unwrap_or(false);
 
@@ -586,7 +616,7 @@ async fn update_app(
 
         if let Some(mut npolicy) = ns.policy {
             npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
-            npolicy.on_behalf_of_email = Some(authed.email);
+            npolicy.on_behalf_of_email = Some(authed.email.clone());
             sqlb.set(
                 "policy",
                 &format!(
@@ -601,18 +631,18 @@ async fn update_app(
         sqlb.returning("path");
 
         let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
-        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?;
+        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
         not_found_if_none(npath_o, "App", path)?
     } else {
         "".to_string()
     };
-    if let Some(nvalue) = &ns.value {
+    let v_id = if let Some(nvalue) = &ns.value {
         let app_id = sqlx::query_scalar!(
             "SELECT id FROM app WHERE path = $1 AND workspace_id = $2",
             npath,
             w_id
         )
-        .fetch_one(&mut *tx)
+        .fetch_one(&mut tx)
         .await?;
 
         let v_id = sqlx::query_scalar!(
@@ -624,7 +654,7 @@ async fn update_app(
             serde_json::to_string(&nvalue).unwrap(),
             authed.username,
         )
-        .fetch_one(&mut *tx)
+        .fetch_one(&mut tx)
         .await?;
 
         sqlx::query!(
@@ -633,20 +663,23 @@ async fn update_app(
             npath,
             w_id
         )
-        .execute(&mut *tx)
+        .execute(&mut tx)
         .await?;
-    }
+        Some(v_id)
+    } else {
+        None
+    };
 
     sqlx::query!(
         "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
         path,
         &w_id
     )
-    .execute(&mut *tx)
+    .execute(&mut tx)
     .await?;
 
     audit_log(
-        &mut *tx,
+        &mut tx,
         &authed.username,
         "apps.update",
         ActionKind::Update,
@@ -655,7 +688,39 @@ async fn update_app(
         None,
     )
     .await?;
-    tx.commit().await?;
+
+    if let Some(v_id) = v_id {
+        let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
+            PushIsolationLevel::Transaction(tx);
+
+        let (dependency_job_uuid, tx) = push(
+            &db,
+            tx,
+            &w_id,
+            JobPayload::AppDependencies { path: npath.clone(), version: v_id },
+            serde_json::Map::new(),
+            &authed.username,
+            &authed.email,
+            windmill_common::users::username_to_permissioned_as(&authed.username),
+            None,
+            None,
+            None,
+            None,
+            None,
+            false,
+            false,
+            None,
+            true,
+            None,
+            None,
+            None,
+        )
+        .await?;
+        tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
+        tx.commit().await?;
+    } else {
+        tx.commit().await?;
+    }
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::UpdateApp {
@@ -709,7 +774,6 @@ async fn execute_component(
     };
 
     let path = path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
 
     let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
         let mut hm = HashMap::new();
@@ -738,7 +802,7 @@ async fn execute_component(
             path,
             &w_id
         )
-        .fetch_optional(&mut tx)
+        .fetch_optional(&db)
         .await?;
 
         let policy = not_found_if_none(policy_o, "App", path)?;
@@ -782,15 +846,16 @@ async fn execute_component(
             (payload, args, None)
         }
         ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
-            let (payload, tag) =
-                get_payload_tag_from_prefixed_path(path, tx.transaction_mut(), &w_id).await?;
+            let (payload, tag) = get_payload_tag_from_prefixed_path(path, &db, &w_id).await?;
             let args = build_args(policy, component, path.to_string(), args)?;
             (payload, args, tag)
         }
         _ => unreachable!(),
     };
+    let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
 
     let (uuid, tx) = push(
+        &db,
         tx,
         &w_id,
         job_payload,
@@ -841,7 +906,7 @@ fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
     Ok((permissioned_as, email))
 }
 
-pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB) -> Result<()> {
+pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> {
     return crate::users::require_is_writer(
         authed,
         path,

backend/windmill-api/src/audit.rs

@@ -12,9 +12,9 @@ use axum::{
     Extension, Json, Router,
 };
 use windmill_audit::{AuditLog, ListAuditLogQuery};
-use windmill_common::{error::JsonResult, utils::Pagination};
+use windmill_common::{db::UserDB, error::JsonResult, utils::Pagination};
 
-use crate::{db::UserDB, users::Authed};
+use crate::db::ApiAuthed;
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -23,7 +23,7 @@ pub fn workspaced_service() -> Router {
 }
 
 async fn get_audit(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(id): Path<i32>,
 ) -> JsonResult<AuditLog> {
@@ -32,7 +32,7 @@ async fn get_audit(
     Ok(Json(audit))
 }
 async fn list_audit(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,

backend/windmill-api/src/capture.rs

@@ -14,14 +14,14 @@ use axum::{
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
 use windmill_common::{
+    db::UserDB,
     error::{JsonResult, Result},
     utils::{not_found_if_none, StripPath},
 };
 
 use crate::{
-    db::{UserDB, DB},
+    db::{ApiAuthed, DB},
     jobs::add_include_headers,
-    users::Authed,
 };
 
 const KEEP_LAST: i64 = 8;
@@ -37,7 +37,7 @@ pub fn global_service() -> Router {
 }
 
 pub async fn new_payload(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<StatusCode> {
@@ -120,7 +120,7 @@ pub async fn update_payload(
 }
 
 pub async fn get_payload(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<serde_json::Value> {

backend/windmill-api/src/db.rs

@@ -6,10 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use sqlx::{Pool, Postgres, Transaction};
-use windmill_common::error::Error;
-
-use crate::users::Authed;
+use sqlx::{Pool, Postgres};
+use windmill_common::{
+    db::{Authable, Authed},
+    error::Error,
+};
 
 pub type DB = Pool<Postgres>;
 
@@ -28,84 +29,58 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
     Ok(())
 }
 
-#[derive(Clone)]
-pub struct UserDB {
-    db: DB,
+#[derive(Clone, Debug)]
+pub struct ApiAuthed {
+    pub email: String,
+    pub username: String,
+    pub is_admin: bool,
+    pub is_operator: bool,
+    pub groups: Vec<String>,
+    // (folder name, can write, is owner)
+    pub folders: Vec<(String, bool, bool)>,
+    pub scopes: Option<Vec<String>>,
 }
 
-impl UserDB {
-    pub fn new(db: DB) -> Self {
-        Self { db }
-    }
-
-    pub async fn begin(
-        self,
-        authed: &Authed,
-    ) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
-        let mut tx = self.db.begin().await?;
-        let user = if authed.is_admin {
-            "windmill_admin"
-        } else {
-            "windmill_user"
-        };
-
-        sqlx::query(&format!("SET LOCAL ROLE {}", user))
-            .execute(&mut *tx)
-            .await?;
-
-        sqlx::query!(
-            "SELECT set_config('session.user', $1, true)",
-            authed.username
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        sqlx::query!(
-            "SELECT set_config('session.groups', $1, true)",
-            &authed.groups.join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        sqlx::query!(
-            "SELECT set_config('session.pgroups', $1, true)",
-            &authed
-                .groups
-                .iter()
-                .map(|x| format!("g/{}", x))
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
-            &authed.folders.clone().into_iter().partition(|x| x.1);
-
-        let mut folders_read = folders_read.clone();
-        folders_read.extend(folders_write.clone());
-        sqlx::query!(
-            "SELECT set_config('session.folders_read', $1, true)",
-            folders_read
-                .iter()
-                .map(|x| x.0.clone())
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        sqlx::query!(
-            "SELECT set_config('session.folders_write', $1, true)",
-            folders_write
-                .iter()
-                .map(|x| x.0.clone())
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        Ok(tx)
+impl From<ApiAuthed> for Authed {
+    fn from(value: ApiAuthed) -> Self {
+        Self {
+            email: value.email,
+            username: value.username,
+            is_admin: value.is_admin,
+            is_operator: value.is_operator,
+            groups: value.groups,
+            folders: value.folders,
+            scopes: value.scopes,
+        }
+    }
+}
+
+impl Authable for ApiAuthed {
+    fn is_admin(&self) -> bool {
+        self.is_admin
+    }
+
+    fn is_operator(&self) -> bool {
+        self.is_operator
+    }
+
+    fn groups(&self) -> &[String] {
+        &self.groups
+    }
+
+    fn folders(&self) -> &[(String, bool, bool)] {
+        &self.folders
+    }
+
+    fn scopes(&self) -> Option<&[std::string::String]> {
+        self.scopes.as_ref().map(|x| x.as_slice())
+    }
+
+    fn email(&self) -> &str {
+        &self.email
+    }
+
+    fn username(&self) -> &str {
+        &self.username
     }
 }

backend/windmill-api/src/drafts.rs

@@ -7,8 +7,8 @@
  */
 
 use crate::{
-    db::{UserDB, DB},
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    db::{ApiAuthed, DB},
+    users::{maybe_refresh_folders, require_owner_of_path},
 };
 
 use axum::{
@@ -18,7 +18,7 @@ use axum::{
 };
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
-use windmill_common::{error::Result, utils::StripPath};
+use windmill_common::{db::UserDB, error::Result, utils::StripPath};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -43,7 +43,7 @@ pub struct Draft {
 }
 
 pub async fn require_writer_of_path(
-    authed: &Authed,
+    authed: &ApiAuthed,
     path: &str,
     w_id: &str,
     db: DB,
@@ -63,7 +63,7 @@ pub async fn require_writer_of_path(
 }
 
 async fn create_draft(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
@@ -95,7 +95,7 @@ async fn create_draft(
 }
 
 async fn delete_draft(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, kind, path)): Path<(String, DraftType, StripPath)>,
 ) -> Result<String> {
@@ -115,7 +115,7 @@ async fn delete_draft(
 }
 
 // async fn get_draft(
-//     authed: Authed,
+//     authed: ApiAuthed,
 //     Extension(user_db): Extension<UserDB>,
 //     Path((w_id, path)): Path<(String, StripPath)>,
 // ) -> JsonResult<Draft> {

backend/windmill-api/src/favorite.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{db::DB, users::Authed};
+use crate::db::{ApiAuthed, DB};
 use axum::{
     extract::{Extension, Path},
     routing::post,
@@ -39,7 +39,7 @@ pub struct Favorite {
 }
 
 async fn star(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(Favorite { favorite_kind, path }): Json<Favorite>,
@@ -58,7 +58,7 @@ async fn star(
 }
 
 async fn unstar(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     Json(Favorite { favorite_kind, path }): Json<Favorite>,

backend/windmill-api/src/flows.rs

@@ -6,10 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use crate::db::ApiAuthed;
 use crate::{
-    db::{UserDB, DB},
+    db::DB,
     schedule::clear_schedule,
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path},
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
@@ -18,6 +19,7 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
+
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
@@ -25,6 +27,7 @@ use sql_builder::SqlBuilder;
 use sqlx::{Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
     flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
     jobs::JobPayload,
@@ -34,7 +37,7 @@ use windmill_common::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
+use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -56,7 +59,7 @@ pub fn global_service() -> Router {
 }
 
 async fn list_flows(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -120,7 +123,7 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
+async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
     let flows = list_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchFlowData?approved=true",
@@ -131,7 +134,7 @@ async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::
 }
 
 async fn list_paths(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
@@ -149,7 +152,7 @@ async fn list_paths(
 }
 
 pub async fn get_hub_flow_by_id(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
     Path(id): Path<i32>,
 ) -> JsonResult<serde_json::Value> {
     let value = http_get_from_hub(
@@ -185,7 +188,7 @@ async fn check_path_conflict<'c>(
 }
 
 async fn create_flow(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
@@ -207,7 +210,7 @@ async fn create_flow(
         w_id,
         nf.path,
         nf.summary,
-        nf.description,
+        nf.description.unwrap_or_else(String::new),
         nf.value,
         &authed.username,
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
@@ -241,12 +244,9 @@ async fn create_flow(
     )
     .await?;
 
-    webhook.send_message(
-        w_id.clone(),
-        WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
-    );
-
+    let tx = PushIsolationLevel::Transaction(tx);
     let (dependency_job_uuid, mut tx) = push(
+        &db,
         tx,
         &w_id,
         JobPayload::FlowDependencies { path: nf.path.clone() },
@@ -279,6 +279,11 @@ async fn create_flow(
     .await?;
     tx.commit().await?;
 
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
+    );
+
     Ok((StatusCode::CREATED, nf.path.to_string()))
 }
 
@@ -305,7 +310,7 @@ async fn check_schedule_conflict<'c>(
     Ok(())
 }
 
-pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB) -> Result<()> {
+pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> {
     return crate::users::require_is_writer(
         authed,
         path,
@@ -318,7 +323,7 @@ pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB)
 }
 
 async fn update_flow(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(db): Extension<DB>,
@@ -347,7 +352,7 @@ async fn update_flow(
          edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL, tag = $9 WHERE path = $7 AND workspace_id = $8",
         nf.path,
         nf.summary,
-        nf.description,
+        nf.description.unwrap_or_else(String::new),
         nf.value,
         &authed.username,
         schema.and_then(|x| serde_json::to_string(&x).ok()),
@@ -393,7 +398,7 @@ async fn update_flow(
         clear_schedule(tx.transaction_mut(), &schedule.path, true, &w_id).await?;
 
         if schedule.enabled {
-            tx = push_scheduled_job(tx, schedule).await?;
+            tx = push_scheduled_job(&db, tx, schedule).await?;
         }
     }
 
@@ -430,7 +435,10 @@ async fn update_flow(
         },
     );
 
+    let tx = PushIsolationLevel::Transaction(tx);
+
     let (dependency_job_uuid, mut tx) = push(
+        &db,
         tx,
         &w_id,
         JobPayload::FlowDependencies { path: nf.path.clone() },
@@ -473,7 +481,7 @@ async fn update_flow(
 }
 
 async fn get_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<Flow> {
@@ -508,7 +516,7 @@ pub struct FlowWDraft {
 }
 
 async fn get_flow_by_path_w_draft(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<FlowWDraft> {
@@ -518,7 +526,7 @@ async fn get_flow_by_path_w_draft(
     let flow_o = sqlx::query_as::<_, FlowWDraft>(
         "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft, flow.tag FROM flow
         LEFT JOIN draft ON 
-        flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' 
+        flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' 
         WHERE flow.path = $1 AND flow.workspace_id = $2",
     )
     .bind(path)
@@ -555,7 +563,7 @@ struct Archived {
 }
 
 async fn archive_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -593,7 +601,7 @@ async fn archive_flow_by_path(
 }
 
 async fn delete_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -706,6 +714,7 @@ mod tests {
                         modules: vec![],
                         skip_failures: true,
                         parallel: false,
+                        parallelism: None,
                     },
                     stop_after_if: Some(StopAfterIf {
                         expr: "previous.isEmpty()".to_string(),
@@ -740,6 +749,10 @@ mod tests {
                 timeout: None,
             }),
             same_worker: false,
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            skip_expr: None,
+            cache_ttl: None,
         };
         let expect = serde_json::json!({
           "modules": [

backend/windmill-api/src/folders.rs

@@ -8,9 +8,11 @@
 
 use std::sync::Arc;
 
+use crate::db::ApiAuthed;
+
 use crate::{
-    db::{UserDB, DB},
-    users::{AuthCache, Authed, Tokened},
+    db::DB,
+    users::{AuthCache, Tokened},
     webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -22,6 +24,7 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
     error::{self, to_anyhow, JsonResult, Result},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, paginate, Pagination},
@@ -74,7 +77,7 @@ pub struct Owner {
 }
 
 async fn list_folders(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -96,7 +99,7 @@ async fn list_folders(
     Ok(Json(rows))
 }
 async fn list_foldernames(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -145,7 +148,7 @@ lazy_static! {
 }
 
 async fn create_folder(
-    authed: Authed,
+    authed: ApiAuthed,
     Tokened { token }: Tokened,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
@@ -163,7 +166,7 @@ async fn create_folder(
     check_name_conflict(&mut tx, &w_id, &ng.name).await?;
     cache.invalidate(&w_id, token).await;
     let owner = username_to_permissioned_as(&authed.username);
-    let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
+    let owners = &ng.owners.unwrap_or_else(|| vec![owner.clone()]);
 
     if let Some(extra_perms) = ng.extra_perms.clone() {
         for o in owners {
@@ -220,13 +223,13 @@ async fn create_folder(
 }
 
 pub async fn is_owner_api(
-    authed: Authed,
+    authed: ApiAuthed,
     Path((_w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
     Ok(Json(is_owner(&authed, &name)))
 }
 
-pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
+pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool {
     if *is_admin {
         true
     } else {
@@ -234,7 +237,7 @@ pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
     }
 }
 
-pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
+pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> {
     if is_owner(authed, name) {
         Ok(())
     } else {
@@ -246,7 +249,7 @@ pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
 }
 
 async fn update_folder(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -343,7 +346,7 @@ pub async fn get_folderopt<'c>(
 }
 
 async fn get_folder(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<Folder> {
@@ -365,7 +368,7 @@ struct FolderUsage {
     pub variables: i64,
 }
 async fn get_folder_usage(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<FolderUsage> {
@@ -446,7 +449,7 @@ async fn get_folder_usage(
 }
 
 async fn delete_folder(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -483,7 +486,7 @@ async fn delete_folder(
 }
 
 async fn add_owner(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -550,7 +553,7 @@ pub async fn get_folders_for_user(
 }
 
 async fn remove_owner(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,

backend/windmill-api/src/granular_acls.rs

@@ -6,18 +6,18 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed},
-};
+use crate::{db::DB, users::require_owner_of_path};
 use axum::{
     extract::{Extension, Path},
     routing::{get, post},
     Json, Router,
 };
 
+use crate::db::ApiAuthed;
+
 use serde::{Deserialize, Serialize};
 use windmill_common::{
+    db::UserDB,
     error::{Error, JsonResult, Result},
     utils::{not_found_if_none, StripPath},
 };
@@ -36,7 +36,7 @@ pub struct GranularAcl {
 }
 
 async fn add_granular_acl(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -83,7 +83,7 @@ async fn add_granular_acl(
 }
 
 async fn remove_granular_acl(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -135,7 +135,7 @@ async fn remove_granular_acl(
 }
 
 async fn get_granular_acls(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<serde_json::Value> {

backend/windmill-api/src/groups.rs

@@ -6,18 +6,16 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{
-    db::{UserDB, DB},
-    users::{get_groups_for_user, Authed},
-    utils::require_super_admin,
-};
+use crate::db::ApiAuthed;
+use crate::{db::DB, users::get_groups_for_user, utils::require_super_admin};
+
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
     Json, Router,
 };
 use windmill_audit::{audit_log, ActionKind};
-use windmill_common::users::username_to_permissioned_as;
+use windmill_common::{db::UserDB, users::username_to_permissioned_as};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     utils::{not_found_if_none, paginate, Pagination},
@@ -113,7 +111,7 @@ struct QueryListGroup {
     pub only_member_of: Option<bool>,
 }
 async fn list_group_names(
-    Authed { username, email, .. }: Authed,
+    ApiAuthed { username, email, .. }: ApiAuthed,
     Extension(db): Extension<DB>,
     Query(QueryListGroup { only_member_of }): Query<QueryListGroup>,
     Path(w_id): Path<String>,
@@ -158,7 +156,7 @@ async fn check_name_conflict<'c>(
 }
 
 pub async fn is_owner(
-    Authed { username, is_admin, groups, .. }: Authed,
+    ApiAuthed { username, is_admin, groups, .. }: ApiAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
@@ -202,8 +200,21 @@ pub async fn require_is_owner(
     }
 }
 
+async fn _check_nb_of_groups(db: &DB) -> Result<()> {
+    let nb_groups = sqlx::query_scalar!("SELECT COUNT(*) FROM group_ WHERE name != 'all'",)
+        .fetch_one(db)
+        .await?;
+    if nb_groups.unwrap_or(0) >= 5 {
+        return Err(Error::BadRequest(
+            "You have reached the maximum number of groups without an enterprise license"
+                .to_string(),
+        ));
+    }
+    return Ok(());
+}
 async fn create_group(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(_db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(ng): Json<NewGroup>,
@@ -212,6 +223,9 @@ async fn create_group(
 
     check_name_conflict(&mut tx, &w_id, &ng.name).await?;
 
+    #[cfg(not(feature = "enterprise"))]
+    _check_nb_of_groups(&_db).await?;
+
     sqlx::query!(
         "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)",
         w_id,
@@ -247,12 +261,13 @@ async fn create_group(
 }
 
 async fn create_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Json(ng): Json<NewGroup>,
 ) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
     let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &authed.email).await?;
+
     sqlx::query!(
         "INSERT INTO instance_group (name, summary) VALUES ($1, $2) ON CONFLICT DO NOTHING",
         ng.name,
@@ -277,12 +292,12 @@ async fn create_igroup(
 }
 
 async fn delete_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Json(ng): Json<NewGroup>,
 ) -> Result<String> {
-    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &authed.email).await?;
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
     sqlx::query!("DELETE FROM instance_group WHERE name = $1", ng.name,)
         .execute(&mut *tx)
         .await?;
@@ -319,7 +334,7 @@ pub async fn get_group_opt<'c>(
 }
 
 async fn get_group(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<GroupInfo> {
@@ -339,7 +354,7 @@ async fn get_group(
 
     let members = sqlx::query_scalar!(
         "SELECT  usr.username  
-            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username 
+            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2
             WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
         name,
         w_id
@@ -358,7 +373,7 @@ async fn get_group(
 }
 
 async fn delete_group(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
@@ -406,7 +421,7 @@ async fn delete_group(
 }
 
 async fn update_group(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
@@ -442,7 +457,7 @@ async fn update_group(
 }
 
 async fn add_user(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,
@@ -479,14 +494,14 @@ async fn add_user(
 }
 
 async fn add_user_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(name): Path<String>,
     Json(Email { email }): Json<Email>,
 ) -> Result<String> {
-    let mut tx = db.begin().await?;
+    require_super_admin(&db, &authed.email).await?;
 
-    require_super_admin(&mut tx, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
     let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name)
         .fetch_optional(&mut *tx)
@@ -521,10 +536,10 @@ struct IGroup {
     name: String,
     emails: Option<Vec<String>>,
 }
-async fn list_igroups(authed: Authed, Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
-    let mut tx = db.begin().await?;
+async fn list_igroups(authed: ApiAuthed, Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
-    require_super_admin(&mut tx, &authed.email).await?;
     let groups = sqlx::query_as!(
         IGroup,
         "SELECT name, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name"
@@ -549,15 +564,14 @@ async fn get_igroup(Path(name): Path<String>, Extension(db): Extension<DB>) -> J
 }
 
 async fn remove_user_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(name): Path<String>,
     Json(Email { email }): Json<Email>,
 ) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
     let mut tx = db.begin().await?;
 
-    require_super_admin(&mut tx, &authed.email).await?;
-
     let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,)
         .fetch_optional(&mut *tx)
         .await?;
@@ -587,7 +601,7 @@ async fn remove_user_igroup(
 }
 
 async fn remove_user(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, name)): Path<(String, String)>,

backend/windmill-api/src/inputs.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
+use crate::{db::ApiAuthed, jobs::CompletedJob};
 use axum::{
     extract::{Path, Query},
     routing::{get, post},
@@ -21,6 +21,7 @@ use std::{
     vec,
 };
 use windmill_common::{
+    db::UserDB,
     error::JsonResult,
     jobs::JobKind,
     scripts::to_i64,
@@ -102,7 +103,7 @@ pub struct Input {
 }
 
 async fn get_input_history(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -115,7 +116,7 @@ async fn get_input_history(
     let sql = &format!(
         "select * from completed_job \
         where {} = $1 and job_kind = $2 and workspace_id = $3 \
-        order by args, started_at desc limit $4 offset $5",
+        order by created_at desc limit $4 offset $5",
         r.runnable_type.column_name()
     );
 
@@ -158,7 +159,7 @@ async fn get_input_history(
 }
 
 async fn list_saved_inputs(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(pagination): Query<Pagination>,
@@ -209,7 +210,7 @@ pub struct CreateInput {
 }
 
 async fn create_input(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Query(r): Query<RunnableParams>,
@@ -245,7 +246,7 @@ pub struct UpdateInput {
 }
 
 async fn update_input(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
     Json(input): Json<UpdateInput>,
@@ -266,7 +267,7 @@ async fn update_input(
 }
 
 async fn delete_input(
-    authed: Authed,
+    authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
     Path((w_id, i_id)): Path<(String, Uuid)>,
 ) -> JsonResult<String> {