feat(frontend): wip

* fix(frontend): fix modifier text size

* fix(frontend): fix modifier text size

.env

@@ -1,11 +1,8 @@
 DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
 
-WM_IMAGE=ghcr.io/windmill-labs/windmill:main
-WM_LICENSE_KEY=""
-
-# For Enterprise Edition, comment the 2 lines above and uncomment below
+# For Enterprise Edition, use:
 # WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
-# WM_LICENSE_KEY="<id>.<expiry>.<signature>"
+WM_IMAGE=ghcr.io/windmill-labs/windmill:main
 
 
 # To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started

.github/workflows/automerge-dependabot.yml

@@ -1,26 +0,0 @@
-name: dependabot auto-merge
-
-on: pull_request_target
-
-permissions:
-  contents: read
-  pull-requests: read
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.6.0
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Enable auto-merge for Dependabot PRs
-        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
-        run: |
-          echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
-          gh pr review --approve "$PR_URL"
-          gh pr merge --auto --squash "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/automerge-dependabot.yml.archived

@@ -0,0 +1,26 @@
+# name: dependabot auto-merge
+
+# on: pull_request_target
+
+# permissions:
+#   contents: read
+#   pull-requests: read
+
+# jobs:
+#   dependabot:
+#     runs-on: ubuntu-latest
+#     if: ${{ github.actor == 'dependabot[bot]' }}
+#     steps:
+#       - name: Dependabot metadata
+#         id: metadata
+#         uses: dependabot/fetch-metadata@v1.6.0
+#         with:
+#           github-token: "${{ secrets.GITHUB_TOKEN }}"
+#       - name: Enable auto-merge for Dependabot PRs
+#         if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
+#         run: |
+#           echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
+#           gh pr review --approve "$PR_URL"
+#           gh pr merge --auto --squash "$PR_URL"
+#         env:
+#           PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/docker-image.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
-  contents: read
+  contents: write
   id-token: write
   packages: write
 
@@ -153,7 +153,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 18
-      - uses: shrink/actions-docker-extract@v2
+      - uses: shrink/actions-docker-extract@v3
         id: extract
         with:
           image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
@@ -165,6 +165,97 @@ jobs:
           bucket: windmill-frontend
           bucket-region: us-east-1
 
+
+  attach_amd64_binary_to_release:
+    needs: [build, build_ee]
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    env:
+      ARCH: amd64
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: |
+          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - run: |
+          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
+          # Also useful to visually check that the arch is the right opencontainers
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          path: "/usr/src/app/windmill"
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-ee
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/usr/src/app/windmill"
+
+      - name: Rename binary with corresponding architecture
+        run: |
+          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
+          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
+
+
+      - name: Attach binary to release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{ steps.extract.outputs.destination }}/*
+            ${{ steps.extract-ee.outputs.destination }}/*
+
+  attach_arm64_binary_to_release:
+    needs: [build, build_ee]
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    env:
+      ARCH: arm64
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: |
+          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - run: |
+          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
+          # Also useful to visually check that the arch is the right opencontainers
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          path: "/usr/src/app/windmill"
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-ee
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/usr/src/app/windmill"
+
+      - name: Rename binary with corresponding architecture
+        run: |
+          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
+          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
+
+
+      - name: Attach binary to release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{ steps.extract.outputs.destination }}/*
+            ${{ steps.extract-ee.outputs.destination }}/*
+
   publish_ecr:
     needs: [build_ee]
     runs-on: ubuntu-latest

.github/workflows/sign-cla.yml

@@ -12,7 +12,7 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: cla-assistant/github-action@v2.2.1
+        uses: cla-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}

CHANGELOG.md

@@ -1,6 +1,153 @@
 # Changelog
 
 
+## [1.183.0](https://github.com/windmill-labs/windmill/compare/v1.182.3...v1.183.0) (2023-10-11)
+
+
+### Features
+
+* **frontend:** Table wizard ([#2416](https://github.com/windmill-labs/windmill/issues/2416)) ([6f0cda0](https://github.com/windmill-labs/windmill/commit/6f0cda0e1ea84e2b5c5d297c841749dc5bae879d))
+
+
+### Bug Fixes
+
+* benchmark config syntax error ([#2432](https://github.com/windmill-labs/windmill/issues/2432)) ([109c2f1](https://github.com/windmill-labs/windmill/commit/109c2f17d68e0cac2f365297cc2fcdd54d9d105a))
+* **frontend:** add a validation for base url ([#2434](https://github.com/windmill-labs/windmill/issues/2434)) ([c914ac6](https://github.com/windmill-labs/windmill/commit/c914ac64cfbaacaf5fe3c7486ea9901ce4828387))
+* **frontend:** fix drawer title truncate ([#2429](https://github.com/windmill-labs/windmill/issues/2429)) ([46d2c13](https://github.com/windmill-labs/windmill/commit/46d2c13e0d2dde1e87c3bbe7cc2be29de84fa2cf))
+* **frontend:** fix mobile multi select ([#2431](https://github.com/windmill-labs/windmill/issues/2431)) ([cb2b6df](https://github.com/windmill-labs/windmill/commit/cb2b6dfdba8953a3d1f432e4af2b2725f5e267ca))
+* **frontend:** fix table wizards for old apps ([#2435](https://github.com/windmill-labs/windmill/issues/2435)) ([e088ec5](https://github.com/windmill-labs/windmill/commit/e088ec566958079e468b3c1f5df057f6e70dffc3))
+
+## [1.182.3](https://github.com/windmill-labs/windmill/compare/v1.182.2...v1.182.3) (2023-10-10)
+
+
+### Bug Fixes
+
+* improve binary build ([094539f](https://github.com/windmill-labs/windmill/commit/094539ff3aa79531953f82941337bdd3d34db630))
+
+## [1.182.2](https://github.com/windmill-labs/windmill/compare/v1.182.1...v1.182.2) (2023-10-10)
+
+
+### Bug Fixes
+
+* add binaries to release ([17b42e6](https://github.com/windmill-labs/windmill/commit/17b42e6a3555ae1f45d8f24934f290a72e3d60c5))
+
+## [1.182.1](https://github.com/windmill-labs/windmill/compare/v1.182.0...v1.182.1) (2023-10-10)
+
+
+### Bug Fixes
+
+* Small fixes UI & Slack OAuth tuto ([#2398](https://github.com/windmill-labs/windmill/issues/2398)) ([e1eccc2](https://github.com/windmill-labs/windmill/commit/e1eccc2d9331ba4e33019a6109bc0368d718397c))
+
+## [1.182.0](https://github.com/windmill-labs/windmill/compare/v1.181.0...v1.182.0) (2023-10-10)
+
+
+### Features
+
+* add support for aggrid ee ([c4a817a](https://github.com/windmill-labs/windmill/commit/c4a817aeb6590d8972342f815f3cf3b891ea1446))
+* **frontend:** App polish ([#2397](https://github.com/windmill-labs/windmill/issues/2397)) ([11e0bc7](https://github.com/windmill-labs/windmill/commit/11e0bc76c4bc80339e43590f5becf6b2442a2227))
+* **frontend:** column definition helper ([#2399](https://github.com/windmill-labs/windmill/issues/2399)) ([53447f1](https://github.com/windmill-labs/windmill/commit/53447f1b43e897bb8856106cabc502c822052441))
+* **frontend:** error handler tutorial ([#2404](https://github.com/windmill-labs/windmill/issues/2404)) ([bc1ad3b](https://github.com/windmill-labs/windmill/commit/bc1ad3b8d09fb2b6547dbcb37ac074ffdf9b383c))
+* **frontend:** fix css editor + fix dark mode ([#2409](https://github.com/windmill-labs/windmill/issues/2409)) ([2d7712c](https://github.com/windmill-labs/windmill/commit/2d7712c02115006fe84cb323b3b3af99ac14ffdb))
+* manage cache and init scripts from worker group UI ([#2396](https://github.com/windmill-labs/windmill/issues/2396)) ([2c9ae41](https://github.com/windmill-labs/windmill/commit/2c9ae41706edc6570559d7d83864fb05c846c0c1))
+
+
+### Bug Fixes
+
+* add lsp absolute imports for deno in all cases ([27c45e3](https://github.com/windmill-labs/windmill/commit/27c45e38cc57350df193440aa0c09ddbca93902a))
+* fix aggrid initialization ([9b75e33](https://github.com/windmill-labs/windmill/commit/9b75e33887c3a9c4cac84d648763a6e3b4490fae))
+* **frontend:** Fix tutorial trigger ([#2392](https://github.com/windmill-labs/windmill/issues/2392)) ([cad37bc](https://github.com/windmill-labs/windmill/commit/cad37bc6defa7a42b96fec6ad0a9bcac55d88d51))
+* improve flow status viewer for large values ([64c5590](https://github.com/windmill-labs/windmill/commit/64c5590aa32e4dbff6af43e711cb6899c02e4ee3))
+* improve handling of large results by frontend ([21454a7](https://github.com/windmill-labs/windmill/commit/21454a7a052db3cc1d24fd36c4504098751c66d2))
+* tarball for workspace export is generated in /tmp/windmill ([f4957d6](https://github.com/windmill-labs/windmill/commit/f4957d66b9bf6124ad3f73912f32cd1ea47b46e2))
+
+## [1.181.0](https://github.com/windmill-labs/windmill/compare/v1.180.0...v1.181.0) (2023-10-05)
+
+
+### Features
+
+* add npm_config_registry support for bun, deno and being settable from UI ([#2373](https://github.com/windmill-labs/windmill/issues/2373)) ([c42b875](https://github.com/windmill-labs/windmill/commit/c42b8750f1d41c9b4de6c96f1ea82239c5325495))
+* **frontend:** add driverjs ([#2327](https://github.com/windmill-labs/windmill/issues/2327)) ([bda6f1f](https://github.com/windmill-labs/windmill/commit/bda6f1fe5d44a3c1d925c1b8a8e872d9f5fba484))
+
+
+### Bug Fixes
+
+* add numeric, array and date types ([#2379](https://github.com/windmill-labs/windmill/issues/2379)) ([768f972](https://github.com/windmill-labs/windmill/commit/768f972cbf578b3394f89120d172b02bcaac5413))
+* add reserved variables in args ([#2371](https://github.com/windmill-labs/windmill/issues/2371)) ([e7165f3](https://github.com/windmill-labs/windmill/commit/e7165f3357a2ba7a690accd78a03c2518aa61860))
+* ai flow prompt fix + explanation in ui ([#2374](https://github.com/windmill-labs/windmill/issues/2374)) ([66d15f0](https://github.com/windmill-labs/windmill/commit/66d15f0c17698077c5bf299af8368e9cfdbf3ecb))
+* flow trigger prompt + lower temp ([#2377](https://github.com/windmill-labs/windmill/issues/2377)) ([733bfe3](https://github.com/windmill-labs/windmill/commit/733bfe3f14e6eb0237c6a528ab64ae71082a4679))
+* **frontend:** fix flow tutorials ([#2383](https://github.com/windmill-labs/windmill/issues/2383)) ([63ad53f](https://github.com/windmill-labs/windmill/commit/63ad53fa70c4f1769d873bff962bfb2d66081163))
+* schema autocomplete/ai ([#2372](https://github.com/windmill-labs/windmill/issues/2372)) ([9ed748a](https://github.com/windmill-labs/windmill/commit/9ed748a0dac95f152f91de6e25b63d841af0dd50))
+* trigger bun prompt ([#2368](https://github.com/windmill-labs/windmill/issues/2368)) ([fc9adbe](https://github.com/windmill-labs/windmill/commit/fc9adbe56081065fa3de662e664fcebe0f4c25ee))
+
+## [1.180.0](https://github.com/windmill-labs/windmill/compare/v1.179.1...v1.180.0) (2023-10-01)
+
+
+### Features
+
+* code content search ([#2367](https://github.com/windmill-labs/windmill/issues/2367)) ([fb96059](https://github.com/windmill-labs/windmill/commit/fb960594fce265d5d4f4eb443e0c9cc19d14e025))
+
+
+### Bug Fixes
+
+* improve connection in apps ([a2fca17](https://github.com/windmill-labs/windmill/commit/a2fca17ae2ac8257154e2aec4a0ceabfe16fc46a))
+
+## [1.179.1](https://github.com/windmill-labs/windmill/compare/v1.179.0...v1.179.1) (2023-09-30)
+
+
+### Bug Fixes
+
+* fix 0 len flow module processing ([f97289a](https://github.com/windmill-labs/windmill/commit/f97289a3d8bc6ce978d0be1fec35a424211e4a20))
+
+## [1.179.0](https://github.com/windmill-labs/windmill/compare/v1.178.1...v1.179.0) (2023-09-30)
+
+
+### Features
+
+* add trustedDependencies escape hatch for bun ([#2364](https://github.com/windmill-labs/windmill/issues/2364)) ([52df265](https://github.com/windmill-labs/windmill/commit/52df2650ea5d5c03e94c96af0b8a79275856fc37))
+* ai code completion ([#2361](https://github.com/windmill-labs/windmill/issues/2361)) ([0937706](https://github.com/windmill-labs/windmill/commit/093770692ac40b8ee0139f24d63bcccda9bf6ddb))
+* **backend:** parse expires_in from string in TokenResponse ([#2353](https://github.com/windmill-labs/windmill/issues/2353)) ([4621915](https://github.com/windmill-labs/windmill/commit/46219154de07ef5a6e071f1c2859cea35c7f9943))
+* **frontend:** copy schema from json and past runs in flow inputs ([#2352](https://github.com/windmill-labs/windmill/issues/2352)) ([3cb2977](https://github.com/windmill-labs/windmill/commit/3cb29778dd70199d9504aa7c1a12bfd7a02569d6))
+
+
+### Bug Fixes
+
+* error handler does not recover flow anymore and error handler is called only once up the flow ([445bf96](https://github.com/windmill-labs/windmill/commit/445bf965eddc6da39a125fce60b53e0903698664))
+* **frontend:** Properly handle click ([#2351](https://github.com/windmill-labs/windmill/issues/2351)) ([55b7f98](https://github.com/windmill-labs/windmill/commit/55b7f982c2bbbb5d4daa9752ec8ffc0c79c374fc))
+* **frontend:** timezone fix ([#2360](https://github.com/windmill-labs/windmill/issues/2360)) ([dcfa5fc](https://github.com/windmill-labs/windmill/commit/dcfa5fc0e40f5cd8dba5a26be31695ce765c7e23))
+* improve superadmin settings page ([b029027](https://github.com/windmill-labs/windmill/commit/b029027c1c75c0b6489966371db7d2f9c99d15f8))
+* non skipped failures stop even in presence of an error handler ([1c5cc0c](https://github.com/windmill-labs/windmill/commit/1c5cc0c237101caf6c5e6e34b11c967a27cd4112))
+* remove shared http clients in rest runtime ([4931ed9](https://github.com/windmill-labs/windmill/commit/4931ed95c4b12f63effa1dd7d6a5cd526a612302))
+
+## [1.178.1](https://github.com/windmill-labs/windmill/compare/v1.178.0...v1.178.1) (2023-09-28)
+
+
+### Bug Fixes
+
+* improve license key check ([035bad5](https://github.com/windmill-labs/windmill/commit/035bad5268d182af3f30915b5356defd7f6ccbc0))
+
+## [1.178.0](https://github.com/windmill-labs/windmill/compare/v1.177.1...v1.178.0) (2023-09-28)
+
+
+### Features
+
+* **frontend:** add app groups management ([#2347](https://github.com/windmill-labs/windmill/issues/2347)) ([20e0427](https://github.com/windmill-labs/windmill/commit/20e0427a1303c1c32f41b198cd2d0f7f28b5bd32))
+* **frontend:** add AppDrawer controls ([#2339](https://github.com/windmill-labs/windmill/issues/2339)) ([3de6d44](https://github.com/windmill-labs/windmill/commit/3de6d446f281dcaac288deee19342a08e0ccf9af))
+* **frontend:** Switch to component list when deleting a component ([#2346](https://github.com/windmill-labs/windmill/issues/2346)) ([6fcd72c](https://github.com/windmill-labs/windmill/commit/6fcd72c79453dd9d60ca869cd9996cc0c25971fa))
+
+
+### Bug Fixes
+
+* add env tags to default worker group ([#2348](https://github.com/windmill-labs/windmill/issues/2348)) ([f5bed95](https://github.com/windmill-labs/windmill/commit/f5bed95ab15bc397f822b06816c43b4b13a84af3))
+
+## [1.177.1](https://github.com/windmill-labs/windmill/compare/v1.177.0...v1.177.1) (2023-09-26)
+
+
+### Bug Fixes
+
+* **frontend:** fix modal closing issues ([#2340](https://github.com/windmill-labs/windmill/issues/2340)) ([18cf8fa](https://github.com/windmill-labs/windmill/commit/18cf8faec16d496e4b327505b682459ed518a5b4))
+* **frontend:** fix overflow ([#2341](https://github.com/windmill-labs/windmill/issues/2341)) ([2e8f2ec](https://github.com/windmill-labs/windmill/commit/2e8f2ec724f6802170121f4f8aa73b697a39c9ee))
+* improve list component handling of non array data ([dc44b08](https://github.com/windmill-labs/windmill/commit/dc44b0841af17227160b9d56ec446e6646a8ab0d))
+
 ## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
 
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:buster-slim as nsjail
+FROM debian:bookworm-slim as nsjail
 
 WORKDIR /nsjail
 
@@ -6,23 +6,23 @@ ARG nsjail=""
 
 RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
-    bison=2:3.3.* \
+    bison=2:3.8.* \
     flex=2.6.* \
-    g++=4:8.3.* \
-    gcc=4:8.3.* \
-    git=1:2.20.* \
-    libprotobuf-dev=3.6.* \
-    libnl-route-3-dev=3.4.* \
-    make=4.2.* \
-    pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*; fi
+    g++=4:12.2.* \
+    gcc=4:12.2.* \
+    git=1:2.39.* \
+    libprotobuf-dev=3.21.* \
+    libnl-route-3-dev=3.7.* \
+    make=4.3-4.1 \
+    pkg-config=1.8.* \
+    protobuf-compiler=3.21.*; fi
 
 
 RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
 RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
-FROM rust:slim-buster AS rust_base
+FROM rust:slim-bookworm AS rust_base
 
 RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
@@ -87,7 +87,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:buster-slim as downloader
+FROM debian:bookworm-slim as downloader
 
 ARG TARGETPLATFORM
 
@@ -101,7 +101,7 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol
 
 RUN unzip deno.zip && rm deno.zip
 
-FROM python:3.11.4-slim-buster
+FROM python:3.11.4-slim-bookworm
 
 ARG TARGETPLATFORM
 
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:1.0.2 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.4 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

README.md

@@ -64,6 +64,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
     - [OAuth for self-hosting](#oauth-for-self-hosting)
     - [smtp for self-hosting](#smtp-for-self-hosting)
     - [Resource types](#resource-types)
+    - [Manually fetch latest Windmill binary](#manually-fetch-latest-windmill-binary)
   - [Environment Variables](#environment-variables)
   - [Run a local dev setup](#run-a-local-dev-setup)
     - [only Frontend](#only-frontend)
@@ -226,11 +227,21 @@ From there, you can follow the setup app and create other users.
 We publish helm charts at:
 <https://github.com/windmill-labs/windmill-helm-charts>.
 
-### Postgres without superuser
+### Run from binaries
 
-If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
-postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for Windmill.
+Each release includes the corresponding binaries for x86_64. You can simply download the 
+latest `windmill` binary using the following set of bash commands.
+```bash
+BINARY_NAME='windmill-amd64' # or windmill-ee-amd64 for the enterprise edition
+LATEST_RELEASE=$(curl -L -s -H 'Accept: application/json' https://github.com/windmill-labs/windmill/releases/latest)
+LATEST_VERSION=$(echo $LATEST_RELEASE | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+ARTIFACT_URL="https://github.com/windmill-labs/windmill/releases/download/$LATEST_VERSION/$BINARY_NAME"
+wget "$ARTIFACT_URL" -O windmill
+```
+
+### OAuth, SSO & SMTP
+
+Windmill Community Edition allows to configure the OAuth, SSO (including Google Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin settings. Do note that there is a limit of 50 SSO users on the community edition.
 
 ### Commercial license
 
@@ -249,91 +260,21 @@ your current infrastructure to Windmill, support with tight SLA, and our global
 cache sync for high-performance/no dependency cache miss of cluster from 10+
 nodes to 200+ nodes.
 
-### OAuth for self-hosting
-
-To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
-the following format:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    "allowed_domains": ["windmill.dev"] //restrict a client OAuth login to some domains
-  }
-}
-```
-
-and mount it at `/usr/src/app/oauth.json`.
-
-The redirect url for the oauth clients is:
-`<instance_url>/user/login_callback/<client>`
-
-Even if you setup oauth, you will still want to **login as admin@windmill.dev /
-changeme** to setup your instance as a super-admin and give yourself admin
-rights.
-
-[The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
-
-To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://www.windmill.dev/docs/misc/contributing). We
-welcome contributions!
-
-You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    // To add a new OAuth2 IdP
-    "login_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "userinfo_url": "<userinfo endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    },
-    // To add a new OAuth2 Resource
-    "connect_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    }
-  }
-}
-```
-
-### smtp for self-hosting
-
-For users to receive emails when you invite them to workspaces or add them to
-the instances using their emails, configure the SMTP env variables in the
-servers:
-
-```
-SMTP_FROM=noreply@windmill.dev
-SMTP_HOST=smtp.gmail.com
-SMTP_PORT=587
-SMTP_USERNAME=ruben@windmill.dev
-SMTP_PASSWORD=yourpasswordapp
-```
-
 ### Resource types
 
 You will also want to import all the approved resource types from
 [WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
 it being synced automatically everyday.
 
+
 ## Environment Variables
 
 | Environment Variable name                     | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
 | --------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL                                  |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| WORKER_GROUP                                  | default                                    | The worker group the worker belongs to and get its configuration pulled from                                                                                                                       | Worker                |
 | SERVER_BIND_ADDR                              | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
 | PORT                                          | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |
 | METRICS_ADDR                                  | None                                       | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                      | All                   |
 | JSON_FMT                                      | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
@@ -373,6 +314,7 @@ it being synced automatically everyday.
 | PIP_LOCAL_DEPENDENCIES                        | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
 | ADDITIONAL_PYTHON_PATHS                       | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
 | INCLUDE_HEADERS                               | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
@@ -395,12 +337,13 @@ it being synced automatically everyday.
 | SMTP_USERNAME                                 | None                                       | username for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
-| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
+| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                       | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
 | GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |
 | SECRET_SALT                                   | None                                       | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server    | Server + Worker       |
 | OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI API base path (no trailing slash)                                                                                                                                                     | Server                |
+| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
 
 ## Run a local dev setup
 

backend/.sqlx/query-025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM global_settings WHERE name = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0"
+}

backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = $1 WHERE workspace_id = $2",
+  "query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Varchar",
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "d7d368f9ef2587c6abd3daaec93401615a53e3d7d949b65ebf01ef8760ba46b8"
+  "hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304"
 }

backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json

@@ -37,6 +37,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597"

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -67,6 +67,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json

@@ -28,6 +28,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"

backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json

@@ -42,6 +42,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT $1 OFFSET $2",
+  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -49,5 +49,5 @@
       true
     ]
   },
-  "hash": "77ba7207c8f5fd7156542cfd9943aa9a9fa87a652131c261f5020bab9ba6b5a3"
+  "hash": "5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c"
 }

backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json

@@ -1,14 +1,15 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = NULL WHERE workspace_id = $1",
+  "query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "28eef409e9f25e4c037dd716d6515e96e8962acb59d5e64f9cc3227e0f5fd058"
+  "hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03"
 }

backend/.sqlx/query-644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e"
+}

backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json

@@ -42,6 +42,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json

@@ -46,6 +46,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86"
+}

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -0,0 +1,66 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e"
+}

backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json

@@ -60,6 +60,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108"
+}

backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT progress::bigint FROM tutorial_progress WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "progress",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2"
+}

backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json

@@ -42,6 +42,7 @@
                 "bash",
                 "postgresql",
                 "nativets",
+                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "openai_resource_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      false
+    ]
+  },
+  "hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee"
+}

backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07"
+}

backend/.sqlx/query-efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb"
+}

backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.177.0"
+version = "1.183.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.177.0"
+version = "1.183.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 

backend/migrations/20230927215813_add_app_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230927215813_add_app_groups.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20230929022202_tutorials.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE tutorial_progress

backend/migrations/20230929022202_tutorials.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE  tutorial_progress(
+    email VARCHAR(255) PRIMARY KEY,
+    progress bit(64) NOT NULL DEFAULT B'0'
+);
+
+GRANT ALL ON tutorial_progress TO windmill_admin;
+GRANT ALL ON tutorial_progress TO windmill_user;

backend/migrations/20230929092213_copilot_code_completion.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN code_completion_enabled;

backend/migrations/20230929092213_copilot_code_completion.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN code_completion_enabled BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20231002123723_notify_delete_config.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231002123723_notify_delete_config.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+
+CREATE FUNCTION "notify_global_setting_delete" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', OLD.name::text);
+    RETURN OLD;
+END;
+$$ LANGUAGE PLPGSQL;
+  
+CREATE OR REPLACE TRIGGER "notify_global_setting_delete"
+ AFTER DELETE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_delete" ();
+

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -38,6 +38,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "gitlab" => "python-gitlab",
     "smbclient" => "smbprotocol",
     "playhouse" => "peewee",
+    "dns" => "dnspython"
 };
 
 fn replace_import(x: String) -> String {

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -48,7 +48,7 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
 }
 
 lazy_static::lazy_static! {
-    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+))?"#).unwrap();
+    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+(?:\[\])?))?"#).unwrap();
 
     // -- $1 name (type) = default
     static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
@@ -213,27 +213,35 @@ pub fn parse_mysql_typ(typ: &str) -> Typ {
 }
 
 pub fn parse_pg_typ(typ: &str) -> Typ {
-    match typ {
-        "varchar" => Typ::Str(None),
-        "text" => Typ::Str(None),
-        "int" => Typ::Int,
-        "bigint" => Typ::Int,
-        "bool" => Typ::Bool,
-        "char" => Typ::Str(None),
-        "smallint" => Typ::Int,
-        "smallserial" => Typ::Int,
-        "serial" => Typ::Int,
-        "bigserial" => Typ::Int,
-        "real" => Typ::Float,
-        "double precision" => Typ::Float,
-        "oid" => Typ::Int,
-        _ => Typ::Str(None),
+    if typ.ends_with("[]") {
+        let base_typ = parse_pg_typ(typ.strip_suffix("[]").unwrap());
+        Typ::List(Box::new(base_typ))
+    } else {
+        match typ {
+            "varchar" => Typ::Str(None),
+            "text" => Typ::Str(None),
+            "int" => Typ::Int,
+            "bigint" => Typ::Int,
+            "bool" => Typ::Bool,
+            "char" => Typ::Str(None),
+            "smallint" => Typ::Int,
+            "smallserial" => Typ::Int,
+            "serial" => Typ::Int,
+            "bigserial" => Typ::Int,
+            "real" => Typ::Float,
+            "double precision" => Typ::Float,
+            "numeric" => Typ::Float,
+            "decimal" => Typ::Float,
+            "oid" => Typ::Int,
+            "date" | "time" | "timestamp" => Typ::Datetime,
+            _ => Typ::Str(None),
+        }
     }
 }
 
 pub fn parse_bigquery_typ(typ: &str) -> Typ {
     if typ.ends_with("[]") {
-        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap_or(typ));
+        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap());
         Typ::List(Box::new(base_typ))
     } else {
         match typ {

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.154.3",
+  "version": "1.180.0",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -97,6 +97,15 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
 function addHeapObject(obj) {
     if (heap_next === heap.length) heap.push(heap.length + 1);
     const idx = heap_next;
@@ -115,15 +124,6 @@ function getFloat64Memory0() {
     return cachedFloat64Memory0;
 }
 
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
-}
-
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -492,13 +492,13 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
-    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
+    imports.wbg.__wbg_eval_2e5c54091872c19c = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -507,6 +507,10 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -543,10 +547,6 @@ function __wbg_get_imports() {
         const ret = BigInt.asUintN(64, arg0);
         return addHeapObject(ret);
     };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;

backend/src/main.rs

@@ -21,7 +21,8 @@ use tokio::{
 };
 use windmill_common::{
     global_settings::{
-        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, LICENSE_KEY_SETTING, OAUTH_SETTING,
+        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, EXTRA_PIP_INDEX_URL_SETTING,
+        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
         REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
     },
     utils::rd_string,
@@ -36,8 +37,9 @@ use windmill_worker::{
 };
 
 use crate::monitor::{
-    initial_load, monitor_db, reload_base_url_setting, reload_license_key,
-    reload_retention_period_setting, reload_server_config, reload_worker_config,
+    initial_load, monitor_db, reload_base_url_setting, reload_extra_pip_index_url_setting,
+    reload_license_key, reload_npm_config_registry_setting, reload_retention_period_setting,
+    reload_server_config, reload_worker_config,
 };
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
@@ -165,6 +167,12 @@ Windmill Community Edition {GIT_VERSION}
         let rsmq2 = rsmq.clone();
         let (port_tx, port_rx) = tokio::sync::oneshot::channel::<u16>();
 
+        DirBuilder::new()
+            .recursive(true)
+            .create("/tmp/windmill")
+            .await
+            .expect("could not create initial server dir");
+
         let server_f = async {
             windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe(), port_tx).await?;
             Ok(()) as anyhow::Result<()>
@@ -196,11 +204,7 @@ Windmill Community Edition {GIT_VERSION}
 
             let mut rx = rx.resubscribe();
             let base_internal_url = base_internal_url.to_string();
-            let rd_delay = rand::thread_rng().gen_range(0..30);
-            tokio::spawn(async move {
-                //monitor_db is applied at start, no need to apply it twice
-                tokio::time::sleep(Duration::from_secs(rd_delay)).await;
-
+            let h = tokio::spawn(async move {
                 let mut listener = retry_listen_pg(&db).await;
 
                 loop {
@@ -220,52 +224,53 @@ Windmill Community Edition {GIT_VERSION}
                                     tracing::info!("Received new pg notification: {n:?}");
                                     match n.channel() {
                                         "notify_config_change" => {
-                                            tracing::info!("Config change detected");
                                             match n.payload() {
                                                 "server" if server_mode => {
-                                                    tracing::info!("Server config change detected");
+                                                    tracing::info!("Server config change detected: {}", n.payload());
+
                                                     reload_server_config(&db).await;
                                                 },
                                                 a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
-                                                    tracing::info!("Worker config change detected");
+                                                    tracing::info!("Worker config change detected: {}", n.payload());
                                                     reload_worker_config(&db, tx.clone(), true).await;
                                                 },
                                                 _ => {
-                                                    ()
+                                                    tracing::debug!("config changed but did not target this server/worker");
                                                 }
                                             }
                                         },
                                         "notify_global_setting_change" => {
-                                            tracing::info!("Global setting change detected");
+                                            tracing::info!("Global setting change detected: {}", n.payload());
                                             match n.payload() {
                                                 BASE_URL_SETTING => {
-                                                    tracing::info!("Base URL setting change detected");
                                                     if let Err(e) = reload_base_url_setting(&db).await {
                                                         tracing::error!(error = %e, "Could not reload base url setting");
                                                     }
                                                 },
                                                 OAUTH_SETTING => {
-                                                    tracing::info!("OAuth setting change detected");
                                                     if let Err(e) = reload_base_url_setting(&db).await {
                                                         tracing::error!(error = %e, "Could not reload oauth setting");
                                                     }
                                                 },
                                                 CUSTOM_TAGS_SETTING => {
-                                                    tracing::info!("Custom tags setting change detected");
                                                     if let Err(e) = reload_custom_tags_setting(&db).await {
                                                         tracing::error!(error = %e, "Could not reload custom tags setting");
                                                     }
                                                 },
                                                 LICENSE_KEY_SETTING => {
-                                                    tracing::info!("License Key setting change detected");
                                                     if let Err(e) = reload_license_key(&db).await {
                                                         tracing::error!(error = %e, "Could not reload license key setting");
                                                     }
                                                 },
                                                 RETENTION_PERIOD_SECS_SETTING => {
-                                                    tracing::info!("Retention period setting change detected");
                                                     reload_retention_period_setting(&db).await
                                                 },
+                                                EXTRA_PIP_INDEX_URL_SETTING => {
+                                                    reload_extra_pip_index_url_setting(&db).await
+                                                },
+                                                NPM_CONFIG_REGISTRY_SETTING => {
+                                                    reload_npm_config_registry_setting(&db).await
+                                                },
                                                 REQUEST_SIZE_LIMIT_SETTING => {
                                                     tracing::info!("Request limit size change detected, killing server expecting to be restarted");
                                                     // we wait a bit randomly to avoid having all servers shutdown at same time
@@ -301,6 +306,9 @@ Windmill Community Edition {GIT_VERSION}
                 }
             });
 
+            if let Err(e) = h.await {
+                tracing::error!("Error waiting for monitor handle:{e}")
+            }
             Ok(()) as anyhow::Result<()>
         };
 

backend/src/monitor.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc};
+use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration};
 
 use once_cell::sync::OnceCell;
 use serde::de::DeserializeOwned;
@@ -15,7 +15,8 @@ use windmill_api::{
 use windmill_common::{
     error,
     global_settings::{
-        BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        BASE_URL_SETTING, EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         RETENTION_PERIOD_SECS_SETTING,
     },
     jobs::{JobKind, QueuedJob},
@@ -25,7 +26,8 @@ use windmill_common::{
     BASE_URL, DB, METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, NPM_CONFIG_REGISTRY,
+    PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
 #[cfg(feature = "enterprise")]
@@ -89,11 +91,9 @@ pub async fn initial_load(
     };
 
     let reload_base_url_f = async {
-        
-            if let Err(e) = reload_base_url_setting(db).await {
-                tracing::error!("Error reloading base url: {:?}", e)
-            }
-        
+        if let Err(e) = reload_base_url_setting(db).await {
+            tracing::error!("Error reloading base url: {:?}", e)
+        }
     };
 
     let reload_server_config_f = async {
@@ -114,11 +114,21 @@ pub async fn initial_load(
     };
 
     let reload_license_key_f = async {
-        if server_mode {
-            #[cfg(feature = "enterprise")]
-            if let Err(e) = reload_license_key(&db).await {
-                tracing::error!("Error reloading license key: {:?}", e)
-            }
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = reload_license_key(&db).await {
+            tracing::error!("Error reloading license key: {:?}", e)
+        }
+    };
+
+    let reload_extra_pip_index_url_f = async {
+        if worker_mode {
+            reload_extra_pip_index_url_setting(&db).await;
+        }
+    };
+
+    let reload_npm_config_registry_f = async {
+        if worker_mode {
+            reload_npm_config_registry_setting(&db).await;
         }
     };
 
@@ -129,7 +139,9 @@ pub async fn initial_load(
         reload_request_size_f,
         reload_base_url_f,
         reload_retention_period_f,
-        reload_license_key_f
+        reload_license_key_f,
+        reload_extra_pip_index_url_f,
+        reload_npm_config_registry_f
     );
 }
 
@@ -205,6 +217,32 @@ pub async fn delete_expired_items(db: &DB) -> () {
     }
 }
 
+pub async fn reload_extra_pip_index_url_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        EXTRA_PIP_INDEX_URL_SETTING,
+        "PIP_EXTRA_INDEX_URL",
+        PIP_EXTRA_INDEX_URL.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading extra_pip_index_url period: {:?}", e)
+    }
+}
+
+pub async fn reload_npm_config_registry_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        NPM_CONFIG_REGISTRY_SETTING,
+        "NPM_CONFIG_REGISTRY",
+        NPM_CONFIG_REGISTRY.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading npm_config_registry period: {:?}", e)
+    }
+}
+
 pub async fn reload_retention_period_setting(db: &DB) {
     if let Err(e) = reload_setting(
         db,
@@ -265,6 +303,44 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> {
     Ok(())
 }
 
+pub async fn reload_option_string_setting(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    lock: Arc<RwLock<Option<String>>>,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var).ok();
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = Some(v)
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        if value.is_none() {
+            tracing::info!("Loaded {setting_name} setting to None");
+        }
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
 pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
     db: &DB,
     setting_name: &str,
@@ -323,12 +399,16 @@ pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     };
 
     let verify_license_key_f = async {
-        if server_mode {
-            #[cfg(feature = "enterprise")]
-            if let Err(e) = verify_license_key().await {
-                tracing::error!("Error verifying license key: {:?}", e);
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = verify_license_key().await {
+            tracing::error!("Error verifying license key: {:?}", e);
+            let mut l = LICENSE_KEY_VALID.write().await;
+            *l = false;
+        } else {
+            let is_valid = LICENSE_KEY_VALID.read().await.clone();
+            if !is_valid {
                 let mut l = LICENSE_KEY_VALID.write().await;
-                *l = false;
+                *l = true;
             }
         }
     };
@@ -387,9 +467,26 @@ pub async fn reload_worker_config(
         let wc = WORKER_CONFIG.read().await;
         let config = config.unwrap();
         if *wc != config {
-            if kill_if_change && (*wc).dedicated_worker != config.dedicated_worker {
-                tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
-                let _ = tx.send(());
+            if kill_if_change {
+                if (*wc).dedicated_worker != config.dedicated_worker {
+                    tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).init_bash != config.init_bash {
+                    tracing::info!("Init bash config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).cache_clear != config.cache_clear {
+                    tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                    tracing::info!("Waiting 5 seconds to allow others workers to start potential jobs that depend on a potential shared cache volume");
+                    tokio::time::sleep(Duration::from_secs(5)).await;
+                    if let Err(e) = windmill_worker::common::clean_cache().await {
+                        tracing::error!("Error cleaning the cache: {e}");
+                    }
+                }
             }
             drop(wc);
 
@@ -408,22 +505,25 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
     .fetch_optional(db)
     .await?;
 
+    let std_base_url = std::env::var("BASE_URL")
+        .ok()
+        .unwrap_or_else(|| "http://localhost".to_string());
     let base_url = if let Some(q) = q_base_url {
         if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
-            v
+            if v != "" {
+                v
+            } else {
+                std_base_url
+            }
         } else {
             tracing::error!(
                 "Could not parse base_url setting as a string, found: {:#?}",
                 &q.value
             );
-            std::env::var("BASE_URL")
-                .ok()
-                .unwrap_or_else(|| "http://localhost".to_string())
+            std_base_url
         }
     } else {
-        std::env::var("BASE_URL")
-            .ok()
-            .unwrap_or_else(|| "http://localhost".to_string())
+        std_base_url
     };
 
     let q_oauth = sqlx::query!(

backend/windmill-api/Cargo.toml

@@ -74,4 +74,5 @@ bytes.workspace = true
 mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
-rsa.workspace = true
+rsa.workspace = true
+uuid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.177.0
+  version: 1.183.0
   title: Windmill API
 
   contact:
@@ -678,6 +678,45 @@ paths:
               schema:
                 type: string
 
+  /users/tutorial_progress:
+    get:
+      summary: get tutorial progress
+      operationId: getTutorialProgress
+      tags:
+        - user
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  progress:
+                    type: integer
+    post:
+      summary: update tutorial progress
+      operationId: updateTutorialProgress
+      tags:
+        - user
+      requestBody:
+        description: progress update
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                progress:
+                  type: integer
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /users/usage:
     get:
       summary: get current usage outside of premium workspaces
@@ -1115,8 +1154,12 @@ paths:
                     type: string
                   openai_resource_path:
                     type: string
+                  code_completion_enabled:
+                    type: boolean
                   error_handler:
                     type: string
+                required:
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/get_deploy_to:
     get:
@@ -1269,24 +1312,28 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/edit_openai_resource_path:
+  /w/{workspace}/workspaces/edit_copilot_config:
     post:
-      summary: edit OpenAI resource path
-      operationId: editOpenaiResourcePath
+      summary: edit copilot config
+      operationId: editCopilotConfig
       tags:
         - workspace
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: WorkspaceOpenaiResourcePath
+        description: WorkspaceCopilotConfig
         required: true
         content:
           application/json:
             schema:
               type: object
+              required:
+                - code_completion_enabled
               properties:
                 openai_resource_path:
                   type: string
+                code_completion_enabled:
+                  type: boolean
       responses:
         "200":
           description: status
@@ -1295,10 +1342,10 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/exists_openai_resource_path:
+  /w/{workspace}/workspaces/get_copilot_info:
     get:
-      summary: OpenAI resource path exists
-      operationId: existsOpenaiResourcePath
+      summary: get copilot info
+      operationId: getCopilotInfo
       tags:
         - workspace
       parameters:
@@ -1310,7 +1357,15 @@ paths:
           content:
             text/plain:
               schema:
-                type: boolean
+                type: object
+                properties:
+                  exists_openai_resource_path:
+                    type: boolean
+                  code_completion_enabled:
+                    type: boolean
+                required:
+                  - exists_openai_resource_path
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/edit_error_handler:
     post:
@@ -2009,6 +2064,12 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/Path"
+        - name: job_id
+          description: job id
+          in: query
+          schema:
+            type: string
+            format: uuid
       responses:
         "200":
           description: resource value
@@ -2079,6 +2140,31 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableResource"
 
+  /w/{workspace}/resources/list_search:
+    get:
+      summary: list resources for search
+      operationId: listSearchResource
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: resource list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/resources/list_names/{name}:
     get:
       summary: list resource names
@@ -2240,6 +2326,65 @@ paths:
                 items:
                   type: string
 
+  /resources/type/hub/list:
+    get:
+      summary: list hub resource types
+      operationId: listHubResourceTypes
+      tags:
+        - resource
+      responses:
+        "200":
+          description: resource type details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                    name:
+                      type: string
+                    schema: {}
+                  required:
+                    - id
+                    - name
+
+  /resources/type/hub/query:
+    get:
+      summary: query hub resource types by similarity
+      operationId: queryHubResourceTypes
+      tags:
+        - resource
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: resource type details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
   /scripts/hub/list:
     get:
       summary: list all available hub scripts
@@ -2501,6 +2646,32 @@ paths:
                   required:
                     - id
 
+  /w/{workspace}/scripts/list_search:
+    get:
+      summary: list scripts for search
+      operationId: listSearchScript
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: script list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    content:
+                      type: string
+                  required:
+                    - path
+                    - content
+
   /w/{workspace}/scripts/list:
     get:
       summary: list all available scripts
@@ -3146,6 +3317,31 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/flows/list_search:
+    get:
+      summary: list flows for search
+      operationId: listSearchFlow
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: flow list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/flows/list:
     get:
       summary: list all available flows
@@ -3449,6 +3645,31 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/apps/list_search:
+    get:
+      summary: list apps for search
+      operationId: listSearchApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: app list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -6843,7 +7064,7 @@ components:
             - "workspaces.edit_deploy_to"
             - "workspaces.edit_auto_invite_domain"
             - "workspaces.edit_webhook"
-            - "workspaces.edit_openai_resource_path"
+            - "workspaces.edit_copilot_config"
             - "workspaces.edit_error_handler"
             - "workspaces.create"
             - "workspaces.update"

backend/windmill-api/src/apps.rs

@@ -15,7 +15,9 @@ use crate::{
     HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Json, Path, Query},
+    response::IntoResponse,
     routing::{delete, get, post},
     Router,
 };
@@ -35,7 +37,7 @@ use windmill_common::{
     jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
+        http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
     },
 };
 use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
@@ -43,6 +45,7 @@ use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
+        .route("/list_search", get(list_search_apps))
         .route("/get/p/*path", get(get_app))
         .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
@@ -160,6 +163,38 @@ pub struct EditApp {
     pub policy: Option<Policy>,
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchApp {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_apps(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchApp>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchApp,
+        "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_apps(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -511,14 +546,19 @@ async fn create_app(
     Ok((StatusCode::CREATED, app.path))
 }
 
-async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchUiData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 pub async fn get_hub_app_by_id(

backend/windmill-api/src/configs.rs

@@ -35,12 +35,7 @@ struct Config {
     config: serde_json::Value,
 }
 
-async fn list_worker_groups(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-) -> error::JsonResult<Vec<Config>> {
-    require_super_admin(&db, &authed.email).await?;
-
+async fn list_worker_groups(Extension(db): Extension<DB>) -> error::JsonResult<Vec<Config>> {
     let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
         .fetch_all(&db)
         .await?;

backend/windmill-api/src/flows.rs

@@ -14,6 +14,8 @@ use crate::{
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
+use axum::body::StreamBody;
+use axum::response::IntoResponse;
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
@@ -24,8 +26,9 @@ use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
-use sqlx::{Postgres, Transaction};
+use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::utils::query_elems_from_hub;
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
@@ -33,15 +36,14 @@ use windmill_common::{
     jobs::JobPayload,
     schedule::Schedule,
     scripts::Schema,
-    utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
-    },
+    utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath},
 };
 use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_flows))
+        .route("/list_search", get(list_search_flows))
         .route("/create", post(create_flow))
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
@@ -58,6 +60,38 @@ pub fn global_service() -> Router {
         .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchFlow {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_flows(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchFlow>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchFlow,
+        "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_flows(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -123,14 +157,19 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchFlowData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 async fn list_paths(

backend/windmill-api/src/jobs.rs

@@ -348,8 +348,12 @@ async fn get_job(
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
 ) -> error::Result<Response> {
-    let cjob_option =
-        sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2")
+    let cjob_option = sqlx::query("SELECT 
+        id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path, 
+        args, CASE WHEN pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
+        schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
+        raw_lock, email, visible_to_owner, mem_peak, tag 
+        FROM completed_job WHERE id = $1 AND workspace_id = $2")
             .bind(id)
             .bind(&w_id)
             .fetch_optional(&db)
@@ -784,46 +788,45 @@ async fn list_jobs(
         return Err(error::Error::BadRequest(
             "cannot specify both success and running".to_string(),
         ));
-
     }
-    let sqlc = if lq.running.is_none() {  
+    let sqlc = if lq.running.is_none() {
         Some(list_completed_jobs_query(
-        &w_id,
-        per_page + offset,
-        0,
-        &ListCompletedQuery { order_desc: Some(true), ..lqc },
-        &[
-            "'CompletedJob' as typ",
-            "id",
-            "workspace_id",
-            "parent_job",
-            "created_by",
-            "created_at",
-            "started_at",
-            "null as scheduled_for",
-            "null as running",
-            "script_hash",
-            "script_path",
-            "null as args",
-            "duration_ms",
-            "success",
-            "deleted",
-            "canceled",
-            "canceled_by",
-            "job_kind",
-            "schedule_path",
-            "permissioned_as",
-            "is_flow_step",
-            "language",
-            "is_skipped",
-            "email",
-            "visible_to_owner",
-            "null as suspend",
-            "mem_peak",
-            "tag",
-            "null as concurrent_limit",
-            "null as concurrency_time_window_s",
-        ],
+            &w_id,
+            per_page + offset,
+            0,
+            &ListCompletedQuery { order_desc: Some(true), ..lqc },
+            &[
+                "'CompletedJob' as typ",
+                "id",
+                "workspace_id",
+                "parent_job",
+                "created_by",
+                "created_at",
+                "started_at",
+                "null as scheduled_for",
+                "null as running",
+                "script_hash",
+                "script_path",
+                "null as args",
+                "duration_ms",
+                "success",
+                "deleted",
+                "canceled",
+                "canceled_by",
+                "job_kind",
+                "schedule_path",
+                "permissioned_as",
+                "is_flow_step",
+                "language",
+                "is_skipped",
+                "email",
+                "visible_to_owner",
+                "null as suspend",
+                "mem_peak",
+                "tag",
+                "null as concurrent_limit",
+                "null as concurrency_time_window_s",
+            ],
         ))
     } else {
         None
@@ -887,13 +890,13 @@ async fn list_jobs(
         );
 
         if let Some(sqlc) = sqlc {
-        format!(
-            "{} UNION ALL {} LIMIT {} OFFSET {};",
-            &sqlq.subquery()?,
-            &sqlc.subquery()?,
-            per_page,
-            offset
-        )
+            format!(
+                "{} UNION ALL {} LIMIT {} OFFSET {};",
+                &sqlq.subquery()?,
+                &sqlc.subquery()?,
+                per_page,
+                offset
+            )
         } else {
             sqlq.query()?
         }
@@ -2674,7 +2677,7 @@ async fn get_job_update(
         )
         .fetch_optional(&mut *tx)
         .await?;
-        let logs = not_found_if_none(logs, "Job", id.to_string())?;
+        let logs = not_found_if_none(logs, "Job Update", id.to_string())?;
         tx.commit().await?;
         Ok(Json(JobUpdate {
             running: Some(false),

backend/windmill-api/src/lib.rs

@@ -205,6 +205,7 @@ pub async fn run_server(
                 .nest("/workers", workers::global_service())
                 .nest("/configs", configs::global_service())
                 .nest("/scripts", scripts::global_service())
+                .nest("/resources", resources::global_service())
                 .nest("/groups", groups::global_service())
                 .nest("/flows", flows::global_service())
                 .nest("/apps", apps::global_service().layer(cors.clone()))

backend/windmill-api/src/oauth2.rs

@@ -35,6 +35,7 @@ use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::db::UserDB;
 use windmill_common::jobs::JobPayload;
+use windmill_common::more_serde::maybe_number_opt;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
@@ -340,6 +341,8 @@ pub struct SlackTokenResponse {
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct TokenResponse {
     access_token: AccessToken,
+    #[serde(deserialize_with = "maybe_number_opt")]
+    #[serde(default)]
     expires_in: Option<u64>,
     refresh_token: Option<RefreshToken>,
     #[serde(deserialize_with = "helpers::deserialize_space_delimited_vec")]
@@ -656,7 +659,7 @@ pub async fn _refresh_token<'c>(
 
 async fn _exchange_token(client: OClient, refresh_token: &str) -> Result<TokenResponse, Error> {
     let token_json = client
-        .exchange_refresh_token(&RefreshToken::from(refresh_token.clone()))
+        .exchange_refresh_token(&RefreshToken::from(refresh_token))
         .with_client(&HTTP_CLIENT)
         .execute::<serde_json::Value>()
         .await
@@ -931,6 +934,15 @@ async fn slack_command(
     ));
 }
 
+fn transform_name_to_email(x: String) -> String {
+    let r = x.replace(' ', "_");
+    if r.contains('@') {
+        return r;
+    } else {
+        return format!("{r}@windmill.dev");
+    }
+}
+
 #[allow(non_snake_case)]
 async fn login_callback(
     Path(client_name): Path<String>,
@@ -952,11 +964,11 @@ async fn login_callback(
 
     if let Ok(token) = token_res {
         let token = &token.access_token.to_string();
+
         let userinfo_url = client_w_config.userinfo_url.as_ref().ok_or_else(|| {
             Error::BadConfig(format!("Missing userinfo_url in client {client_name}"))
         })?;
         let user = http_get_user_info::<LoginUserInfo>(&HTTP_CLIENT, userinfo_url, token).await?;
-
         let email = match client_name.as_str() {
             "github" => http_get_user_info::<Vec<GHEmailInfo>>(
                 &HTTP_CLIENT,
@@ -971,9 +983,15 @@ async fn login_callback(
             )))?
             .email
             .to_string(),
-            _ => user.email.clone().ok_or_else(|| {
-                error::Error::BadRequest("email address not fetchable from user info".to_string())
-            })?,
+            _ => user
+                .email
+                .clone()
+                .or(user.name.clone().map(transform_name_to_email))
+                .ok_or_else(|| {
+                    error::Error::BadRequest(
+                        "email address not fetchable from user info".to_string(),
+                    )
+                })?,
         }
         .to_lowercase();
 

backend/windmill-api/src/openai.rs

@@ -7,7 +7,6 @@ use crate::{
 use axum::{
     body::{Bytes, StreamBody},
     extract::{Extension, Path},
-    http::HeaderMap,
     response::IntoResponse,
     routing::post,
     Router,
@@ -170,12 +169,8 @@ async fn proxy(
         ));
     }
 
-    let mut headers = HeaderMap::new();
-    for (k, v) in response.headers().iter() {
-        headers.insert(k, v.clone());
-    }
-
     let status_code = response.status();
+    let headers = response.headers().clone();
     let stream = response.bytes_stream();
 
     Ok((status_code, headers, StreamBody::new(stream)))

backend/windmill-api/src/resources.rs

@@ -8,11 +8,14 @@
 
 use crate::{
     db::{ApiAuthed, DB},
-    users::{maybe_refresh_folders, require_owner_of_path},
+    users::{maybe_refresh_folders, require_owner_of_path, Tokened},
     webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Path, Query},
+    response::IntoResponse,
     routing::{delete, get, post},
     Json, Router,
 };
@@ -21,16 +24,28 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{FromRow, Postgres, Transaction};
+use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     db::UserDB,
     error::{Error, JsonResult, Result},
-    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    jobs::QueuedJob,
+    utils::{
+        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
+    },
+    variables,
 };
 
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/type/hub/list", get(list_hub_resource_types))
+        .route("/type/hub/query", get(query_hub_resource_types))
+}
+
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
+        .route("/list_search", get(list_search_resources))
         .route("/list_names/:type", get(list_names))
         .route("/get/*path", get(get_resource))
         .route("/exists/*path", get(exists_resource))
@@ -144,6 +159,37 @@ async fn list_names(
     Ok(Json(rows))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchResource {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_resources(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchResource>> {
+    let mut tx = user_db.begin(&authed).await?;
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchResource,
+        "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_resources(
     authed: ApiAuthed,
     Query(lq): Query<ListResourceQuery>,
@@ -271,10 +317,16 @@ async fn get_resource_value(
     Ok(Json(value))
 }
 
+#[derive(Deserialize)]
+struct JobInfo {
+    job_id: Option<Uuid>,
+}
 async fn get_resource_value_interpolated(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Tokened { token }: Tokened,
     Path((w_id, path)): Path<(String, StripPath)>,
+    Query(job_info): Query<JobInfo>,
 ) -> JsonResult<Option<serde_json::Value>> {
     let path = path.to_path();
     let mut tx = user_db.clone().begin(&authed).await?;
@@ -291,7 +343,7 @@ async fn get_resource_value_interpolated(
     let value = not_found_if_none(value_o, "Resource", path)?;
     if let Some(value) = value {
         Ok(Json(Some(
-            transform_json_value(&authed, &user_db, &w_id, value).await?,
+            transform_json_value(&authed, &user_db, &w_id, value, &job_info.job_id, &token).await?,
         )))
     } else {
         Ok(Json(None))
@@ -306,6 +358,8 @@ pub async fn transform_json_value<'c>(
     user_db: &UserDB,
     workspace: &str,
     v: Value,
+    job_id: &Option<Uuid>,
+    token: &str,
 ) -> Result<Value> {
     match v {
         Value::String(y) if y.starts_with("$var:") => {
@@ -331,16 +385,65 @@ pub async fn transform_json_value<'c>(
             tx.commit().await?;
             let v = not_found_if_none(v, "Resource", path)?;
             if let Some(v) = v {
-                transform_json_value(authed, user_db, workspace, v).await
+                transform_json_value(authed, user_db, workspace, v, job_id, token).await
             } else {
                 Ok(Value::Null)
             }
         }
+        Value::String(y) if y.starts_with("$") && job_id.is_some() => {
+            let mut tx = user_db.clone().begin(authed).await?;
+            let job = sqlx::query_as::<_, QueuedJob>(
+                "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2",
+            )
+            .bind(job_id.unwrap())
+            .bind(workspace)
+            .fetch_optional(&mut *tx)
+            .await?;
+            tx.commit().await?;
+
+            let job = not_found_if_none(job, "Job", job_id.unwrap().to_string())?;
+
+            let flow_path = if let Some(uuid) = job.parent_job {
+                let mut tx: Transaction<'_, Postgres> = user_db.clone().begin(authed).await?;
+                let p = sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid)
+                    .fetch_optional(&mut *tx)
+                    .await?
+                    .flatten();
+                tx.commit().await?;
+                p
+            } else {
+                None
+            };
+
+            let variables = variables::get_reserved_variables(
+                &job.workspace_id,
+                token,
+                &job.email,
+                &job.created_by,
+                &job.id.to_string(),
+                &job.permissioned_as,
+                job.script_path.clone(),
+                job.parent_job.map(|x| x.to_string()),
+                flow_path,
+                job.schedule_path.clone(),
+                job.flow_step_id.clone(),
+            )
+            .await;
+
+            let name = y.strip_prefix("$").unwrap();
+
+            let value = variables
+                .iter()
+                .find(|x| x.name == name)
+                .map(|x| x.value.clone())
+                .unwrap_or_else(|| y);
+            Ok(serde_json::json!(value))
+        }
         Value::Object(mut m) => {
             for (a, b) in m.clone().into_iter() {
                 m.insert(
                     a.clone(),
-                    transform_json_value(authed, user_db, workspace, b).await?,
+                    transform_json_value(authed, user_db, workspace, b, job_id, token).await?,
                 );
             }
             Ok(Value::Object(m))
@@ -806,3 +909,46 @@ async fn update_resource_type(
 
     Ok(format!("resource_type {} updated", name))
 }
+
+async fn list_hub_resource_types(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/resource_types/list",
+        &email,
+        None,
+    )
+    .await?;
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
+}
+
+#[derive(Deserialize)]
+struct HubResourceTypesQuery {
+    text: String,
+    limit: Option<i64>,
+}
+async fn query_hub_resource_types(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Query(query): Query<HubResourceTypesQuery>,
+) -> impl IntoResponse {
+    let mut query_params = vec![("text", query.text)];
+    if let Some(query_limit) = query.limit {
+        query_params.push(("limit", query_limit.to_string().clone()));
+    }
+    let (status_code, headers, response) = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/resource_types/query",
+        &email,
+        Some(query_params),
+    )
+    .await?;
+
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
+}

backend/windmill-api/src/scripts.rs

@@ -14,7 +14,9 @@ use crate::{
     HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Path, Query},
+    response::IntoResponse,
     routing::{get, post},
     Json, Router,
 };
@@ -41,8 +43,7 @@ use windmill_common::{
     },
     users::username_to_permissioned_as,
     utils::{
-        list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
-        Pagination, StripPath,
+        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
 };
 use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
@@ -96,6 +97,7 @@ pub fn global_unauthed_service() -> Router {
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_scripts))
+        .route("/list_search", get(list_search_scripts))
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
         .route("/get/draft/*path", get(get_script_by_path_w_draft))
@@ -111,6 +113,37 @@ pub fn workspaced_service() -> Router {
         .route("/list_paths", get(list_paths))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchScript {
+    path: String,
+    content: String,
+}
+async fn list_search_scripts(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchScript>> {
+    let mut tx = user_db.begin(&authed).await?;
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 10;
+
+    let rows = sqlx::query_as!(
+        SearchScript,
+        "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_scripts(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -201,14 +234,19 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
-async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let asks = list_elems_from_hub(
+async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(asks))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 #[derive(Deserialize)]
@@ -220,17 +258,26 @@ struct HubScriptsQuery {
 async fn query_hub_scripts(
     ApiAuthed { email, .. }: ApiAuthed,
     Query(query): Query<HubScriptsQuery>,
-) -> JsonResult<serde_json::Value> {
-    let asks = query_elems_from_hub(
+) -> impl IntoResponse {
+    let mut query_params = vec![("text", query.text)];
+    if let Some(query_kind) = query.kind {
+        query_params.push(("kind", query_kind.clone()));
+    }
+    if let Some(query_limit) = query.limit {
+        query_params.push(("limit", query_limit.to_string().clone()));
+    }
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/scripts/query",
         &email,
-        &query.text,
-        &query.kind,
-        &query.limit,
+        Some(query_params),
     )
     .await?;
-    Ok(Json(asks))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 fn hash_script(ns: &NewScript) -> i64 {

backend/windmill-api/src/settings.rs

@@ -79,7 +79,6 @@ pub struct TestKey {
     pub license_key: String,
 }
 
-
 pub async fn test_license_key(
     Extension(db): Extension<DB>,
     authed: ApiAuthed,
@@ -109,6 +108,14 @@ pub async fn get_local_settings(
 pub struct Value {
     pub value: serde_json::Value,
 }
+
+pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> {
+    sqlx::query!("DELETE FROM global_settings WHERE name = $1", key,)
+        .execute(db)
+        .await?;
+    tracing::info!("Unset global setting {}", key);
+    Ok(())
+}
 pub async fn set_global_setting(
     Extension(db): Extension<DB>,
     authed: ApiAuthed,
@@ -116,14 +123,25 @@ pub async fn set_global_setting(
     Json(value): Json<Value>,
 ) -> error::Result<()> {
     require_super_admin(&db, &authed.email).await?;
-    sqlx::query!(
-        "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
-        key,
-        value.value
-    )
-    .execute(&db)
-    .await?;
-    tracing::info!("Set global setting {} to {}", key, value.value);
+    match value.value {
+        serde_json::Value::Null => {
+            delete_global_setting(&db, &key).await?;
+        }
+        serde_json::Value::String(x) if x.is_empty() => {
+            delete_global_setting(&db, &key).await?;
+        }
+        v => {
+            sqlx::query!(
+            "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+            key,
+            v
+        )
+        .execute(&db)
+        .await?;
+            tracing::info!("Set global setting {} to {}", key, v);
+        }
+    };
+
     Ok(())
 }
 

backend/windmill-api/src/users.rs

@@ -89,6 +89,10 @@ pub fn global_service() -> Router {
         .route("/usage", get(get_usage))
         .route("/all_runnables", get(get_all_runnables))
         .route("/refresh_token", get(refresh_token))
+        .route(
+            "/tutorial_progress",
+            post(update_tutorial_progress).get(get_tutorial_progress),
+        )
     // .route("/list_invite_codes", get(list_invite_codes))
     // .route("/create_invite_code", post(create_invite_code))
     // .route("/signup", post(signup))
@@ -713,6 +717,9 @@ async fn list_users(
     Extension(user_db): Extension<UserDB>,
     Path(w_id): Path<String>,
 ) -> JsonResult<Vec<UserWithUsage>> {
+    if *CLOUD_HOSTED && w_id == "demo" {
+        require_admin(authed.is_admin, &authed.username)?;
+    }
     let mut tx = user_db.begin(&authed).await?;
     let rows = sqlx::query(
         "
@@ -751,7 +758,7 @@ async fn list_users_as_super_admin(
 
     let rows = sqlx::query_as!(
         GlobalUserInfo,
-        "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT \
+        "SELECT email, login_type::text, verified, super_admin, name, company from password ORDER BY super_admin DESC, email LIMIT \
          $1 OFFSET $2",
         per_page as i32,
         offset as i32
@@ -761,6 +768,40 @@ async fn list_users_as_super_admin(
     Ok(Json(rows))
 }
 
+#[derive(Serialize, Deserialize)]
+struct Progress {
+    progress: u64,
+}
+async fn get_tutorial_progress(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> JsonResult<Progress> {
+    let res = sqlx::query_scalar!(
+        "SELECT progress::bigint FROM tutorial_progress WHERE email = $1",
+        authed.email
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten()
+    .unwrap_or_default() as u64;
+    Ok(Json(Progress { progress: res }))
+}
+
+async fn update_tutorial_progress(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Json(progress): Json<Progress>,
+) -> Result<String> {
+    sqlx::query_scalar!(
+        "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)",
+        progress.progress as i64,
+        authed.email
+    )
+    .execute(&db)
+    .await?;
+    Ok("tutorial progress updated".to_string())
+}
+
 // async fn list_invite_codes(
 //     authed: ApiAuthed,
 //     Extension(db): Extension<DB>,

backend/windmill-api/src/workspaces.rs

@@ -69,8 +69,8 @@ pub fn workspaced_service() -> Router {
         .route("/edit_deploy_to", post(edit_deploy_to))
         .route("/tarball", get(tarball_workspace))
         .route("/premium_info", get(premium_info))
-        .route("/edit_openai_resource_path", post(edit_openai_resource_path))
-        .route("/exists_openai_resource_path", get(exists_openai_resource_path) )
+        .route("/edit_copilot_config", post(edit_copilot_config))
+        .route("/get_copilot_info", get(get_copilot_info) )
         .route("/edit_error_handler", post(edit_error_handler));
 
     #[cfg(feature = "enterprise")]
@@ -125,6 +125,7 @@ pub struct WorkspaceSettings {
     pub webhook: Option<String>,
     pub deploy_to: Option<String>,
     pub openai_resource_path: Option<String>,
+    pub code_completion_enabled: bool,
     pub error_handler: Option<String>,
 }
 
@@ -166,8 +167,9 @@ struct EditWebhook {
 }
 
 #[derive(Deserialize)]
-struct EditOpenaiResourcePath {
+struct EditCopilotConfig {
     openai_resource_path: Option<String>,
+    code_completion_enabled: bool,
 }
 
 #[derive(Deserialize)]
@@ -670,12 +672,12 @@ async fn edit_webhook(
     Ok(format!("Edit webhook for workspace {}", &w_id))
 }
 
-async fn edit_openai_resource_path(
+async fn edit_copilot_config(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
     ApiAuthed { is_admin, username, .. }: ApiAuthed,
-    Json(eo): Json<EditOpenaiResourcePath>,
+    Json(eo): Json<EditCopilotConfig>,
 ) -> Result<String> {
     require_admin(is_admin, &username)?;
 
@@ -683,15 +685,17 @@ async fn edit_openai_resource_path(
 
     if let Some(openai_resource_path) = &eo.openai_resource_path {
         sqlx::query!(
-            "UPDATE workspace_settings SET openai_resource_path = $1 WHERE workspace_id = $2",
+            "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
             openai_resource_path,
+            eo.code_completion_enabled,
             &w_id
         )
         .execute(&mut *tx)
         .await?;
     } else {
         sqlx::query!(
-            "UPDATE workspace_settings SET openai_resource_path = NULL WHERE workspace_id = $1",
+            "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
+            eo.code_completion_enabled,
             &w_id,
         )
         .execute(&mut *tx)
@@ -700,37 +704,43 @@ async fn edit_openai_resource_path(
     audit_log(
         &mut *tx,
         &authed.username,
-        "workspaces.edit_openai_resource_path",
+        "workspaces.edit_copilot_config",
         ActionKind::Update,
         &w_id,
         Some(&authed.email),
-        Some([("openai_resource_path", &format!("{:?}", eo.openai_resource_path)[..])].into()),
+            Some([("openai_resource_path", &format!("{:?}", eo.openai_resource_path)[..]), ("code_completion_enabled", &format!("{:?}", eo.code_completion_enabled)[..])].into()),
     )
     .await?;
     tx.commit().await?;
 
-    Ok(format!("Edit openai_resource_path for workspace {}", &w_id))
+    Ok(format!("Edit copilot config for workspace {}", &w_id))
 }
 
-
-async fn exists_openai_resource_path(
+#[derive(Serialize)]
+struct CopilotInfo {
+    pub exists_openai_resource_path: bool,
+    pub code_completion_enabled: bool,
+}
+async fn get_copilot_info(
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-) -> JsonResult<bool> {
+) -> JsonResult<CopilotInfo> {
 
     let mut tx = db.begin().await?;
-    let openai_resource_path = sqlx::query_scalar!(
-        "SELECT openai_resource_path FROM workspace_settings WHERE workspace_id = $1",
+    let record = sqlx::query!(
+        "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
         &w_id
     )
     .fetch_one(&mut *tx)
     .await
-    .map_err(|e| Error::InternalErr(format!("getting openai_resource_path: {e}")))?;
+    .map_err(|e| Error::InternalErr(format!("getting openai_resource_path and code_completion_enabled: {e}")))?;
     tx.commit().await?;
 
-    let exists = openai_resource_path.is_some();
 
-    Ok(Json(exists))
+    Ok(Json(CopilotInfo {
+        exists_openai_resource_path: record.openai_resource_path.is_some(),
+        code_completion_enabled: record.code_completion_enabled,
+    }))
 }
 
 
@@ -856,7 +866,7 @@ async fn _check_nb_of_workspaces(db: &DB) -> Result<()> {
         .await?;
     if nb_workspaces.unwrap_or(0) >= 3 {
         return Err(Error::BadRequest(
-            "You have reached the maximum number of workspaces (3 outside of default group 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
+            "You have reached the maximum number of workspaces (3 outside of default worskapce 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
                 .to_string(),
         ));
     }
@@ -958,6 +968,12 @@ async fn create_workspace(
     .execute(&mut *tx)
     .await?;
 
+    sqlx::query!(
+        "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+        nw.id,
+    )
+    .execute(&mut *tx)
+    .await?;
 
     sqlx::query!(
         "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
@@ -1476,7 +1492,7 @@ async fn tarball_workspace(
 ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
-    let tmp_dir = TempDir::new_in(".")?;
+    let tmp_dir = TempDir::new_in("/tmp/windmill/")?;
 
     let name = match archive_type.as_deref() {
         Some("tar") | None => Ok(format!("windmill-{w_id}.tar")),
@@ -1680,7 +1696,7 @@ async fn tarball_workspace(
 
     archive.finish().await?;
 
-    let file = tokio::fs::File::open(file_path).await?;
+    let file = tokio::fs::File::open(&file_path).await?;
 
     let stream = ReaderStream::new(file);
     let body = StreamBody::new(stream);
@@ -1692,6 +1708,5 @@ async fn tarball_workspace(
             format!("attachment; filename=\"{name}\""),
         ),
     ];
-
     Ok((headers, body))
 }

backend/windmill-common/src/db.rs

@@ -106,7 +106,7 @@ impl UserDB {
         .await?;
 
         let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
-            &authed.folders().clone().into_iter().partition(|x| x.1);
+            &authed.folders().into_iter().partition(|x| x.1);
 
         let mut folders_read = folders_read.clone();
         folders_read.extend(folders_write.clone());

backend/windmill-common/src/global_settings.rs

@@ -5,6 +5,8 @@ pub const OAUTH_SETTING: &str = "oauths";
 pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
 pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
 pub const LICENSE_KEY_SETTING: &str = "license_key";
+pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
+pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
 
 pub const ENV_SETTINGS: [&str; 54] = [
     "DISABLE_NSJAIL",

backend/windmill-common/src/more_serde.rs

@@ -9,6 +9,8 @@
 //! helpers for serde + serde derive attributes
 
 use crate::utils::rd_string;
+use serde::{Deserialize, Deserializer};
+use std::{fmt::Display, str::FromStr};
 
 pub fn default_true() -> bool {
     true
@@ -33,3 +35,32 @@ pub fn default_id() -> String {
 pub fn is_default<T: Default + std::cmp::PartialEq>(t: &T) -> bool {
     &T::default() == t
 }
+
+pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
+where
+    D: Deserializer<'de>,
+    T: FromStr + serde::Deserialize<'de>,
+    <T as FromStr>::Err: Display,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum NumericOrNull<'a, T> {
+        String(String),
+        Str(&'a str),
+        RawT(T),
+        Null,
+    }
+
+    match NumericOrNull::<T>::deserialize(deserializer)? {
+        NumericOrNull::String(s) => match s.as_str() {
+            "" => Ok(None),
+            _ => T::from_str(&s).map(Some).map_err(serde::de::Error::custom),
+        },
+        NumericOrNull::Str(s) => match s {
+            "" => Ok(None),
+            _ => T::from_str(s).map(Some).map_err(serde::de::Error::custom),
+        },
+        NumericOrNull::RawT(i) => Ok(Some(i)),
+        NumericOrNull::Null => Ok(None),
+    }
+}

backend/windmill-common/src/users.rs

@@ -17,7 +17,11 @@ pub fn username_to_permissioned_as(user: &str) -> String {
 }
 
 pub fn truncate_token(token: &str) -> String {
-    let mut s = token[..10].to_owned();
-    s.push_str("*****");
-    s
+    if token.len() > 10 {
+        let mut s = token[..10].to_owned();
+        s.push_str("*****");
+        s
+    } else {
+        token.to_string()
+    }
 }

backend/windmill-common/src/utils.rs

@@ -7,6 +7,7 @@
  */
 
 use crate::error::{Error, Result};
+use hyper::{HeaderMap, StatusCode};
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
@@ -73,43 +74,18 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
     }
 }
 
-#[cfg(feature = "reqwest")]
-pub async fn list_elems_from_hub(
-    http_client: &reqwest::Client,
-    url: &str,
-    email: &str,
-) -> Result<serde_json::Value> {
-    let rows = http_get_from_hub(http_client, url, email, false, None)
-        .await?
-        .json::<serde_json::Value>()
-        .await
-        .map_err(crate::error::to_anyhow)?;
-    Ok(rows)
-}
-
 #[cfg(feature = "reqwest")]
 pub async fn query_elems_from_hub(
     http_client: &reqwest::Client,
     url: &str,
     email: &str,
-    query_text: &str,
-    query_kind: &Option<String>,
-    query_limit: &Option<i64>,
-) -> Result<serde_json::Value> {
-    let mut query_params = vec![("text", query_text)];
-    if let Some(query_kind) = query_kind {
-        query_params.push(("kind", query_kind.as_str()));
-    }
-    let query_limit = query_limit.unwrap_or(0).to_string();
-    if query_limit.parse::<i64>().unwrap() > 0 {
-        query_params.push(("limit", query_limit.as_str()));
-    }
-    let rows = http_get_from_hub(http_client, url, email, false, Some(query_params))
-        .await?
-        .json::<serde_json::Value>()
-        .await
-        .map_err(crate::error::to_anyhow)?;
-    Ok(rows)
+    query_params: Option<Vec<(&str, String)>>,
+) -> Result<(StatusCode, HeaderMap, reqwest::Response)> {
+    let response = http_get_from_hub(http_client, url, email, false, query_params).await?;
+
+    let status = response.status();
+
+    Ok((status, response.headers().clone(), response))
 }
 
 #[cfg(feature = "reqwest")]
@@ -118,7 +94,7 @@ pub async fn http_get_from_hub(
     url: &str,
     email: &str,
     plain: bool,
-    query_params: Option<Vec<(&str, &str)>>,
+    query_params: Option<Vec<(&str, String)>>,
 ) -> Result<reqwest::Response> {
     let mut request = http_client
         .get(url)

backend/windmill-common/src/worker.rs

@@ -33,6 +33,10 @@ lazy_static::lazy_static! {
     pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
         worker_tags: Default::default(),
         dedicated_worker: Default::default(),
+        cache_clear: Default::default(),
+        init_bash: Default::default(),
+        additional_python_paths: Default::default(),
+        pip_local_dependencies: Default::default()
     }));
 
     pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
@@ -162,6 +166,22 @@ pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
             WorkspacedPath { workspace_id: workspace.to_string(), path: script_path.to_string() }
         }
     });
+    if *WORKER_GROUP == "default" {
+        let mut all_tags = config
+            .worker_tags
+            .unwrap_or_default()
+            .into_iter()
+            .chain(
+                std::env::var("WORKER_TAGS")
+                    .ok()
+                    .map(|x| x.split(',').map(|x| x.to_string()).collect_vec())
+                    .unwrap_or_default(),
+            )
+            .sorted()
+            .collect_vec();
+        all_tags.dedup();
+        config.worker_tags = Some(all_tags);
+    }
     Ok(WorkerConfig {
         worker_tags: config
             .worker_tags
@@ -179,6 +199,26 @@ pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
             })
             .unwrap_or_else(|| DEFAULT_TAGS.clone()),
         dedicated_worker,
+        init_bash: config
+            .init_bash
+            .or_else(|| std::env::var("INIT_SCRIPT").ok())
+            .and_then(|x| if x.is_empty() { None } else { Some(x) }),
+        cache_clear: config.cache_clear,
+        pip_local_dependencies: config.pip_local_dependencies.or_else(|| {
+            let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
+                .ok()
+                .map(|x| x.split(',').map(|x| x.to_string()).collect());
+            if pip_local_dependencies == Some(vec!["".to_string()]) {
+                None
+            } else {
+                pip_local_dependencies
+            }
+        }),
+        additional_python_paths: config.additional_python_paths.or_else(|| {
+            std::env::var("ADDITIONAL_PYTHON_PATHS")
+                .ok()
+                .map(|x| x.split(':').map(|x| x.to_string()).collect())
+        }),
     })
 }
 
@@ -187,15 +227,27 @@ pub struct WorkspacedPath {
     pub workspace_id: String,
     pub path: String,
 }
+
 #[derive(Serialize, Deserialize)]
 pub struct WorkerConfigOpt {
     pub worker_tags: Option<Vec<String>>,
     pub dedicated_worker: Option<String>,
+    pub init_bash: Option<String>,
+    pub cache_clear: Option<u32>,
+    pub additional_python_paths: Option<Vec<String>>,
+    pub pip_local_dependencies: Option<Vec<String>>,
 }
 
 impl Default for WorkerConfigOpt {
     fn default() -> Self {
-        Self { worker_tags: Default::default(), dedicated_worker: Default::default() }
+        Self {
+            worker_tags: Default::default(),
+            dedicated_worker: Default::default(),
+            init_bash: Default::default(),
+            cache_clear: Default::default(),
+            additional_python_paths: Default::default(),
+            pip_local_dependencies: Default::default(),
+        }
     }
 }
 
@@ -203,4 +255,8 @@ impl Default for WorkerConfigOpt {
 pub struct WorkerConfig {
     pub worker_tags: Vec<String>,
     pub dedicated_worker: Option<WorkspacedPath>,
+    pub init_bash: Option<String>,
+    pub cache_clear: Option<u32>,
+    pub additional_python_paths: Option<Vec<String>>,
+    pub pip_local_dependencies: Option<Vec<String>>,
 }

backend/windmill-queue/src/jobs.rs

@@ -215,6 +215,9 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     logs: String,
     rsmq: Option<R>,
 ) -> Result<Uuid, Error> {
+    // tracing::error!("Start");
+    // let start = tokio::time::Instant::now();
+
     let is_flow =
         queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview;
     let duration = if is_flow {
@@ -416,6 +419,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     }
 
     tracing::debug!("Added completed job {}", queued_job.id);
+    // tracing::error!("{:?}", start.elapsed());
+
     Ok(queued_job.id)
 }
 

backend/windmill-worker/Cargo.toml

@@ -72,6 +72,7 @@ sha2 = { workspace = true, optional = true }
 pem = { workspace = true, optional = true }
 urlencoding.workspace = true
 nix.workspace = true
+bytes.workspace = true
 
 [build-dependencies]
 deno_fetch.workspace = true

backend/windmill-worker/loader_builder.bun.ts

@@ -14,7 +14,7 @@ if (!bo.success) {
   process.exit(1);
 } else {
   let content = await fs.readFile("./out/main.js", { encoding: "utf8" });
-  const imports = new Bun.Transpiler().scanImports(content);
+  const imports = new Bun.Transpiler().scanImports(content.replaceAll("__require", "require"));
 
   const { intersect } = require("semver-intersect");
   const dependencies: Record<string, string[]> = {};

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -12,6 +12,7 @@ cwd: "/tmp/bun"
 
 clone_newnet: false
 clone_newuser: {CLONE_NEWUSER}
+clone_newcgroup: false
 
 keep_caps: false
 keep_env: true
@@ -28,6 +29,11 @@ mount {
 	is_bind: true
 }
 
+mount {
+    src: "/sys/fs"
+    dst: "/sys/fs"
+	is_bind: true
+}
 
 mount {
     src: "/lib64"
@@ -132,6 +138,7 @@ mount {
     dst: "/tmp/windmill/cache/bun"
     is_bind: true
     mandatory: false
+    rw: true
 }
 
 {SHARED_MOUNT}

backend/windmill-worker/src/bash_executor.rs

@@ -6,12 +6,13 @@ use serde_json::{json, Value};
 use tokio::process::Command;
 use windmill_common::{error::Error, jobs::QueuedJob};
 
+const BIN_BASH: &str = "/bin/bash";
 const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
 
 use crate::{
     common::{
         get_reserved_variables, handle_child, read_file, read_file_content, set_logs,
-        transform_json_value, write_file,
+        start_child_process, transform_json_value, write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     TZ_ENV,
@@ -53,6 +54,8 @@ pub async fn handle_bash_job(
         &client,
         &job.workspace_id,
         job.args.clone().unwrap_or_else(|| json!({})),
+        job,
+        db,
     )
     .await?
     {
@@ -87,7 +90,8 @@ pub async fn handle_bash_job(
         .await?;
         let mut cmd_args = vec!["--config", "run.config.proto", "--", "/bin/bash", "main.sh"];
         cmd_args.extend(args);
-        Command::new(NSJAIL_PATH.as_str())
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(reserved_variables)
@@ -95,12 +99,13 @@ pub async fn handle_bash_job(
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(cmd_args)
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
     } else {
         let mut cmd_args = vec!["main.sh"];
         cmd_args.extend(&args);
-        Command::new("/bin/bash")
+        let mut bash_cmd = Command::new(BIN_BASH);
+        bash_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -110,8 +115,8 @@ pub async fn handle_bash_job(
             .env("HOME", HOME_ENV.as_str())
             .args(cmd_args)
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(bash_cmd, BIN_BASH).await?
     };
     handle_child(
         &job.id,
@@ -173,6 +178,8 @@ pub async fn handle_powershell_job(
             &client,
             &job.workspace_id,
             job.args.clone().unwrap_or_else(|| json!({})),
+            job,
+            db,
         )
         .await?
         {

backend/windmill-worker/src/bigquery_executor.rs

@@ -55,9 +55,10 @@ pub async fn do_bigquery(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
 ) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };

backend/windmill-worker/src/bun_executor.rs

@@ -8,6 +8,7 @@ use anyhow::Context;
 
 use base64::Engine;
 use itertools::Itertools;
+use regex::Regex;
 use uuid::Uuid;
 
 #[cfg(feature = "enterprise")]
@@ -16,7 +17,7 @@ use crate::{common::build_envs_map, JobCompleted};
 use crate::{
     common::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
-        write_file, write_file_binary,
+        start_child_process, write_file, write_file_binary,
     },
     AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV,
     NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
@@ -42,7 +43,7 @@ use tokio::sync::mpsc::{Receiver, Sender};
 use windmill_common::variables;
 
 use windmill_common::{
-    error::{self, Result},
+    error::{self, to_anyhow, Result},
     jobs::QueuedJob,
 };
 use windmill_parser::Typ;
@@ -56,6 +57,10 @@ const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.conf
 pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
 pub const EMPTY_FILE: &str = "<empty>";
 
+lazy_static::lazy_static! {
+    pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap();
+}
+
 pub async fn gen_lockfile(
     logs: &mut String,
     job_id: &Uuid,
@@ -67,6 +72,7 @@ pub async fn gen_lockfile(
     base_internal_url: &str,
     worker_name: &str,
     export_pkg: bool,
+    trusted_deps: Vec<String>,
 ) -> Result<Option<String>> {
     let _ = write_file(
         &job_dir,
@@ -87,22 +93,23 @@ pub async fn gen_lockfile(
     .await?;
 
     let common_bun_proc_envs: HashMap<String, String> =
-        get_common_bun_proc_envs(&base_internal_url);
+        get_common_bun_proc_envs(&base_internal_url).await;
 
-    let child = Command::new(&*BUN_PATH)
+    let mut child_cmd = Command::new(&*BUN_PATH);
+    child_cmd
         .current_dir(job_dir)
         .env_clear()
         .envs(common_bun_proc_envs.clone())
         .args(vec!["run", "build.ts"])
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, &*BUN_PATH).await?;
 
     handle_child(
         job_id,
         db,
         logs,
-        child,
+        child_process,
         false,
         worker_name,
         w_id,
@@ -112,6 +119,37 @@ pub async fn gen_lockfile(
     )
     .await?;
 
+    if trusted_deps.len() > 0 {
+        logs.push_str(&format!(
+            "\ndetected trustedDependencies: {}\n",
+            trusted_deps.join(", ")
+        ));
+        let mut content = "".to_string();
+        {
+            let mut file = File::open(format!("{job_dir}/package.json")).await?;
+            file.read_to_string(&mut content).await?;
+        }
+        let mut value =
+            serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(&content)
+                .map_err(to_anyhow)?;
+        value.insert(
+            "trustedDependencies".to_string(),
+            serde_json::json!(trusted_deps),
+        );
+        write_file(
+            job_dir,
+            "package.json",
+            &serde_json::to_string(&value).map_err(to_anyhow)?,
+        )
+        .await?;
+
+        let mut content = "".to_string();
+        {
+            let mut file = File::open(format!("{job_dir}/package.json")).await?;
+            file.read_to_string(&mut content).await?;
+        }
+    }
+
     install_lockfile(
         logs,
         job_id,
@@ -156,20 +194,21 @@ pub async fn install_lockfile(
     worker_name: &str,
     common_bun_proc_envs: HashMap<String, String>,
 ) -> Result<()> {
-    let child = Command::new(&*BUN_PATH)
+    let mut child_cmd = Command::new(&*BUN_PATH);
+    child_cmd
         .current_dir(job_dir)
         .env_clear()
         .envs(common_bun_proc_envs)
         .args(vec!["install"])
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, &*BUN_PATH).await?;
 
     handle_child(
         job_id,
         db,
         logs,
-        child,
+        child_process,
         false,
         worker_name,
         w_id,
@@ -181,6 +220,23 @@ pub async fn install_lockfile(
     Ok(())
 }
 
+pub fn get_trusted_deps(code: &str) -> Vec<String> {
+    // postinstall not allowed with nsjail
+    if !*DISABLE_NSJAIL {
+        return vec![];
+    }
+    TRUSTED_DEP
+        .captures(code)
+        .map(|x| {
+            x[1].to_string()
+                .trim()
+                .split(',')
+                .map(|y| y.trim().to_string())
+                .collect_vec()
+        })
+        .unwrap_or_default()
+}
+
 #[tracing::instrument(level = "trace", skip_all)]
 pub async fn handle_bun_job(
     requirements_o: Option<String>,
@@ -196,8 +252,9 @@ pub async fn handle_bun_job(
     shared_mount: &str,
 ) -> error::Result<serde_json::Value> {
     let _ = write_file(job_dir, "main.ts", inner_content).await?;
+
     let common_bun_proc_envs: HashMap<String, String> =
-        get_common_bun_proc_envs(&base_internal_url);
+        get_common_bun_proc_envs(&base_internal_url).await;
 
     if let Some(reqs) = requirements_o {
         let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
@@ -209,16 +266,20 @@ pub async fn handle_bun_job(
         let _ = write_file(job_dir, "package.json", &splitted[0]).await?;
         let lockb = splitted[1];
         if lockb != EMPTY_FILE {
-            let _ = write_file_binary(
-                job_dir,
-                "bun.lockb",
-                &base64::engine::general_purpose::STANDARD
-                    .decode(&splitted[1])
-                    .map_err(|_| {
-                        error::Error::InternalErr("Could not decode bun.lockb".to_string())
-                    })?,
-            )
-            .await?;
+            let has_trusted_deps = &splitted[0].contains("trustedDependencies");
+
+            if !has_trusted_deps {
+                let _ = write_file_binary(
+                    job_dir,
+                    "bun.lockb",
+                    &base64::engine::general_purpose::STANDARD
+                        .decode(&splitted[1])
+                        .map_err(|_| {
+                            error::Error::InternalErr("Could not decode bun.lockb".to_string())
+                        })?,
+                )
+                .await?;
+            }
 
             install_lockfile(
                 logs,
@@ -230,25 +291,41 @@ pub async fn handle_bun_job(
                 common_bun_proc_envs.clone(),
             )
             .await?;
-            remove_dir_all(format!("{}/node_modules", job_dir)).await?;
+            if !has_trusted_deps {
+                remove_dir_all(format!("{}/node_modules", job_dir)).await?;
+            }
+        }
+    } else {
+        // TODO: remove once bun implement a reasonable set of trusted deps
+        let trusted_deps = get_trusted_deps(inner_content);
+        let empty_trusted_deps = trusted_deps.len() == 0;
+        let has_custom_config_registry = common_bun_proc_envs.contains_key("NPM_CONFIG_REGISTRY");
+        if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
+            logs.push_str("\n\n--- BUN INSTALL ---\n");
+            set_logs(&logs, &job.id, &db).await;
+            let _ = gen_lockfile(
+                logs,
+                &job.id,
+                &job.workspace_id,
+                db,
+                &client.get_token().await,
+                &job.script_path(),
+                job_dir,
+                base_internal_url,
+                worker_name,
+                false,
+                trusted_deps,
+            )
+            .await?;
+        }
+
+        if empty_trusted_deps && !has_custom_config_registry {
+            let node_modules_path = format!("{}/node_modules", job_dir);
+            let node_modules_exists = tokio::fs::metadata(&node_modules_path).await.is_ok();
+            if node_modules_exists {
+                remove_dir_all(&node_modules_path).await?;
+            }
         }
-    } else if !*DISABLE_NSJAIL {
-        logs.push_str("\n\n--- BUN INSTALL ---\n");
-        set_logs(&logs, &job.id, &db).await;
-        let _ = gen_lockfile(
-            logs,
-            &job.id,
-            &job.workspace_id,
-            db,
-            &client.get_token().await,
-            &job.script_path(),
-            job_dir,
-            base_internal_url,
-            worker_name,
-            false,
-        )
-        .await?;
-        remove_dir_all(format!("{}/node_modules", job_dir)).await?;
     }
 
     logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
@@ -310,7 +387,7 @@ run().catch(async (e) => {{
     let reserved_variables_args_out_f = async {
         let client = client.get_authed().await;
         let args_and_out_f = async {
-            create_args_and_out_file(&client, job, job_dir).await?;
+            create_args_and_out_file(&client, job, job_dir, db).await?;
             Ok(()) as Result<()>
         };
         let reserved_variables_f = async {
@@ -364,7 +441,8 @@ plugin(p)
         )
         .await?;
 
-        Command::new(NSJAIL_PATH.as_str())
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -384,8 +462,8 @@ plugin(p)
                 "/tmp/bun/wrapper.ts",
             ])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
     } else {
         let script_path = format!("{job_dir}/wrapper.ts");
         let args = vec![
@@ -396,7 +474,8 @@ plugin(p)
             "./loader.bun.ts",
             &script_path,
         ];
-        Command::new(&*BUN_PATH)
+        let mut bun_cmd = Command::new(&*BUN_PATH);
+        bun_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -404,8 +483,8 @@ plugin(p)
             .envs(common_bun_proc_envs)
             .args(args)
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(bun_cmd, &*BUN_PATH).await?
     };
 
     handle_child(
@@ -424,8 +503,8 @@ plugin(p)
     read_result(job_dir).await
 }
 
-pub fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
-    let mut deno_envs: HashMap<String, String> = HashMap::from([
+pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
+    let mut bun_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
         (String::from("HOME"), HOME_ENV.clone()),
         (String::from("TZ"), TZ_ENV.clone()),
@@ -443,10 +522,10 @@ pub fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, Stri
         ),
     ]);
 
-    if let Some(ref s) = *NPM_CONFIG_REGISTRY {
-        deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
+    if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
+        bun_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
     }
-    return deno_envs;
+    return bun_envs;
 }
 
 #[cfg(feature = "enterprise")]
@@ -472,7 +551,7 @@ pub async fn start_worker(
     let mut logs = "".to_string();
     let _ = write_file(job_dir, "main.ts", inner_content).await?;
     let common_bun_proc_envs: HashMap<String, String> =
-        get_common_bun_proc_envs(&base_internal_url);
+        get_common_bun_proc_envs(&base_internal_url).await;
     let context = variables::get_reserved_variables(
         w_id,
         &token,
@@ -521,6 +600,7 @@ pub async fn start_worker(
         )
         .await?;
     } else if !*DISABLE_NSJAIL {
+        let trusted_deps = get_trusted_deps(inner_content);
         logs.push_str("\n\n--- BUN INSTALL ---\n");
         let _ = gen_lockfile(
             &mut logs,
@@ -533,6 +613,7 @@ pub async fn start_worker(
             base_internal_url,
             worker_name,
             false,
+            trusted_deps,
         )
         .await?;
     }
@@ -643,7 +724,8 @@ plugin(p)
             "./loader.bun.ts",
             &script_path,
         ];
-        Command::new(&*BUN_PATH)
+        let mut bun_cmd = Command::new(&*BUN_PATH);
+        bun_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(context_envs)
@@ -658,8 +740,8 @@ plugin(p)
             .args(args)
             .stdin(Stdio::piped())
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(bun_cmd, &*BUN_PATH).await?
     };
 
     let stdout = child

backend/windmill-worker/src/common.rs

@@ -3,6 +3,7 @@ use nix::sys::signal::{self, Signal};
 use nix::unistd::Pid;
 use serde_json::{json, Value};
 use sqlx::{Pool, Postgres};
+use tokio::process::Command;
 use tokio::{fs::File, io::AsyncReadExt};
 use windmill_api_client::{types::CreateResource, Client};
 use windmill_common::worker::CLOUD_HOSTED;
@@ -40,7 +41,8 @@ use futures::{
 };
 
 use crate::{
-    AuthedClient, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, TIMEOUT_DURATION, WHITELIST_ENVS,
+    AuthedClient, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, ROOT_CACHE_DIR, TIMEOUT_DURATION,
+    WHITELIST_ENVS,
 };
 
 #[tracing::instrument(level = "trace", skip_all)]
@@ -48,9 +50,10 @@ pub async fn create_args_and_out_file(
     client: &AuthedClient,
     job: &QueuedJob,
     job_dir: &str,
+    db: &Pool<Postgres>,
 ) -> Result<(), Error> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), job, db).await?)
     } else {
         None
     };
@@ -84,6 +87,8 @@ pub async fn transform_json_value(
     client: &AuthedClient,
     workspace: &str,
     v: Value,
+    job: &QueuedJob,
+    db: &Pool<Postgres>,
 ) -> error::Result<Value> {
     match v {
         Value::String(y) if y.starts_with("$var:") => {
@@ -104,16 +109,50 @@ pub async fn transform_json_value(
             }
             Ok(client
                 .get_client()
-                .get_resource_value_interpolated(workspace, path)
+                .get_resource_value_interpolated(workspace, path, Some(&job.id))
                 .await
                 .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))?
                 .into_inner())
         }
+        Value::String(y) if y.starts_with("$") => {
+            let flow_path = if let Some(uuid) = job.parent_job {
+                sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid)
+                    .fetch_optional(db)
+                    .await?
+                    .flatten()
+            } else {
+                None
+            };
+
+            let variables = variables::get_reserved_variables(
+                &job.workspace_id,
+                &client.token,
+                &job.email,
+                &job.created_by,
+                &job.id.to_string(),
+                &job.permissioned_as,
+                job.script_path.clone(),
+                job.parent_job.map(|x| x.to_string()),
+                flow_path,
+                job.schedule_path.clone(),
+                job.flow_step_id.clone(),
+            )
+            .await;
+
+            let name = y.strip_prefix("$").unwrap();
+
+            let value = variables
+                .iter()
+                .find(|x| x.name == name)
+                .map(|x| x.value.clone())
+                .unwrap_or_else(|| y);
+            Ok(json!(value))
+        }
         Value::Object(mut m) => {
             for (a, b) in m.clone().into_iter() {
                 m.insert(
                     a.clone(),
-                    transform_json_value(&a, client, workspace, b).await?,
+                    transform_json_value(&a, client, workspace, b, job, &db).await?,
                 );
             }
             Ok(Value::Object(m))
@@ -129,13 +168,27 @@ pub async fn read_file_content(path: &str) -> error::Result<String> {
     Ok(content)
 }
 pub async fn read_file(path: &str) -> error::Result<serde_json::Value> {
-    let content = read_file_content(path).await?;
-    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
-        return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
+    // tracing::error!("START1");
+    // let start = Instant::now();
+
+    let r = if *CLOUD_HOSTED {
+        let content = read_file_content(path).await?;
+        if content.len() > MAX_RESULT_SIZE {
+            return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
         If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
-    }
-    serde_json::from_str(&content)
-        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
+        };
+        serde_json::from_str(&content)
+            .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
+    } else {
+        let file = std::fs::File::open(path)
+            .map_err(|e| error::Error::ExecutionErr(format!("Error opening file {path}: {e}")))?;
+        let reader = std::io::BufReader::new(file);
+
+        serde_json::from_reader(reader)
+            .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
+    };
+    // tracing::error!("{:?}", start.elapsed());
+    return r;
 }
 pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
     return read_file(&format!("{job_dir}/result.json")).await;
@@ -543,6 +596,12 @@ pub async fn handle_child(
     }
 }
 
+pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result<Child, Error> {
+    return cmd
+        .spawn()
+        .map_err(|err| tentatively_improve_error(Error::IoErr(err), executable));
+}
+
 /// takes stdout and stderr from Child, panics if either are not present
 ///
 /// builds a stream joining both stdout and stderr each read line by line
@@ -642,3 +701,20 @@ async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef<str>, db: impl Borrow<
         tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}");
     }
 }
+
+fn tentatively_improve_error(err: Error, executable: &str) -> Error {
+    if err
+        .to_string()
+        .contains("No such file or directory (os error 2)")
+    {
+        return Error::InternalErr(format!("Executable {executable} not found on worker"));
+    }
+    return err;
+}
+
+pub async fn clean_cache() -> error::Result<()> {
+    tracing::info!("Started cleaning cache");
+    tokio::fs::remove_dir_all(ROOT_CACHE_DIR).await?;
+    tracing::info!("Finished cleaning cache");
+    Ok(())
+}

backend/windmill-worker/src/deno_executor.rs

@@ -6,7 +6,7 @@ use uuid::Uuid;
 use crate::{
     common::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
-        write_file,
+        start_child_process, write_file,
     },
     AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
     NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
@@ -63,7 +63,7 @@ async fn get_common_deno_proc_envs(
         ),
     ]);
 
-    if let Some(ref s) = *NPM_CONFIG_REGISTRY {
+    if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
         deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
     }
     return deno_envs;
@@ -92,7 +92,12 @@ pub async fn generate_deno_lock(
     write_file(job_dir, "import_map.json", &import_map).await?;
     write_file(job_dir, "empty.ts", "").await?;
 
-    let child = Command::new(DENO_PATH.as_str())
+    let mut deno_envs = HashMap::new();
+    if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
+        deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
+    }
+    let mut child_cmd = Command::new(DENO_PATH.as_str());
+    child_cmd
         .current_dir(job_dir)
         .args(vec![
             "cache",
@@ -103,15 +108,16 @@ pub async fn generate_deno_lock(
             &import_map_path,
             "main.ts",
         ])
+        .envs(deno_envs)
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, DENO_PATH.as_str()).await?;
 
     handle_child(
         job_id,
         db,
         logs,
-        child,
+        child_process,
         false,
         worker_name,
         w_id,
@@ -243,7 +249,7 @@ run().catch(async (e) => {{
     let reserved_variables_args_out_f = async {
         let client = client.get_authed().await;
         let args_and_out_f = async {
-            create_args_and_out_file(&client, job, job_dir).await?;
+            create_args_and_out_file(&client, job, job_dir, db).await?;
             Ok(()) as Result<()>
         };
         let reserved_variables_f = async {
@@ -263,7 +269,10 @@ run().catch(async (e) => {{
         write_import_map_f
     )?;
 
-    let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
+    let mut common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
+    if !*DISABLE_NSJAIL {
+        common_deno_proc_envs.insert("HOME".to_string(), job_dir.to_string());
+    }
 
     //do not cache local dependencies
     let reload = format!("--reload={base_internal_url}");
@@ -297,7 +306,8 @@ run().catch(async (e) => {{
             args.push("-A");
         }
         args.push(&script_path);
-        Command::new(DENO_PATH.as_str())
+        let mut deno_cmd = Command::new(DENO_PATH.as_str());
+        deno_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -306,8 +316,8 @@ run().catch(async (e) => {{
             .env("DENO_DIR", DENO_CACHE_DIR)
             .args(args)
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(deno_cmd, DENO_PATH.as_str()).await?
     };
     // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
     // start = Instant::now();

backend/windmill-worker/src/go_executor.rs

@@ -17,7 +17,7 @@ use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE};
 use crate::{
     common::{
         capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result,
-        set_logs, write_file,
+        set_logs, start_child_process, write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
@@ -90,7 +90,7 @@ pub async fn handle_go_job(
 
         logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
         set_logs(logs, &job.id, db).await;
-        create_args_and_out_file(client, job, job_dir).await?;
+        create_args_and_out_file(client, job, job_dir, db).await?;
         {
             let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
 
@@ -177,7 +177,8 @@ func Run(req Req) (interface{{}}, error){{
             }
         }
 
-        let build_go = Command::new(GO_PATH.as_str())
+        let mut build_go_cmd = Command::new(GO_PATH.as_str());
+        build_go_cmd
             .current_dir(job_dir)
             .env_clear()
             .env("PATH", PATH_ENV.as_str())
@@ -186,13 +187,13 @@ func Run(req Req) (interface{{}}, error){{
             .env("HOME", HOME_ENV.as_str())
             .args(vec!["build", "main.go"])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
+            .stderr(Stdio::piped());
+        let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?;
         handle_child(
             &job.id,
             db,
             logs,
-            build_go,
+            build_go_process,
             false,
             worker_name,
             &job.workspace_id,
@@ -212,7 +213,7 @@ func Run(req Req) (interface{{}}, error){{
         tokio::fs::copy(path, format!("{job_dir}/main")).await?;
         logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
         set_logs(logs, &job.id, db).await;
-        create_args_and_out_file(client, job, job_dir).await?;
+        create_args_and_out_file(client, job, job_dir, db).await?;
     }
 
     let reserved_variables = get_reserved_variables(job, &client.token, db).await?;
@@ -228,7 +229,8 @@ func Run(req Req) (interface{{}}, error){{
                 .replace("{SHARED_MOUNT}", shared_mount),
         )
         .await?;
-        Command::new(NSJAIL_PATH.as_str())
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -238,10 +240,11 @@ func Run(req Req) (interface{{}}, error){{
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
     } else {
-        let mut run_go = Command::new("./main");
+        let compiled_executable_name = "./main";
+        let mut run_go = Command::new(compiled_executable_name);
         run_go
             .current_dir(job_dir)
             .env_clear()
@@ -260,10 +263,8 @@ func Run(req Req) (interface{{}}, error){{
             run_go.env("GOPROXY", goproxy);
         }
 
-        run_go
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+        run_go.stdout(Stdio::piped()).stderr(Stdio::piped());
+        start_child_process(run_go, compiled_executable_name).await?
     };
     handle_child(
         &job.id,
@@ -316,18 +317,19 @@ pub async fn install_go_dependencies(
 ) -> error::Result<String> {
     if !skip_go_mod {
         gen_go_mymod(code, job_dir).await?;
-        let child = Command::new("go")
+        let mut child_cmd = Command::new(GO_PATH.as_str());
+        child_cmd
             .current_dir(job_dir)
             .args(vec!["mod", "init", "mymod"])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?;
+            .stderr(Stdio::piped());
+        let child_process = start_child_process(child_cmd, GO_PATH.as_str()).await?;
 
         handle_child(
             job_id,
             db,
             logs,
-            child,
+            child_process,
             false,
             worker_name,
             w_id,
@@ -377,18 +379,20 @@ pub async fn install_go_dependencies(
     }
 
     let mod_command = if skip_tidy { "download" } else { "tidy" };
-    let child = Command::new(GO_PATH.as_str())
+    let mut child_cmd = Command::new(GO_PATH.as_str());
+    child_cmd
         .current_dir(job_dir)
         .env("GOPATH", GO_CACHE_DIR)
         .args(vec!["mod", mod_command])
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, GO_PATH.as_str()).await?;
+
     handle_child(
         job_id,
         db,
         logs,
-        child,
+        child_process,
         false,
         worker_name,
         &w_id,

backend/windmill-worker/src/graphql_executor.rs

@@ -31,9 +31,10 @@ pub async fn do_graphql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
 ) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };

backend/windmill-worker/src/js_eval.rs

@@ -27,7 +27,6 @@ use tokio::{
 };
 use uuid::Uuid;
 use windmill_common::{error::Error, flow_status::JobResult};
-use windmill_queue::HTTP_CLIENT;
 
 use crate::AuthedClient;
 
@@ -403,7 +402,7 @@ async fn op_resource(
     if let Some(client) = client {
         let result = client
             .get_client()
-            .get_resource_value_interpolated(&client.workspace, path)
+            .get_resource_value_interpolated(&client.workspace, path, None)
             .await?;
         Ok(result.into_inner())
     } else {
@@ -505,7 +504,8 @@ pub async fn eval_fetch_timeout(
                 let op_state = js_runtime.op_state();
                 let mut op_state = op_state.borrow_mut();
                 op_state.put(PermissionsContainer{});
-                op_state.put(HTTP_CLIENT.clone());
+                //reqwest client seems to not be sharable between runtimes unfortunately
+                // op_state.put(HTTP_CLIENT.clone());
                 op_state.put(MainArgs { args: spread });
                 op_state.put(LogString { s: String::new() });
             }

backend/windmill-worker/src/lib.rs

@@ -5,7 +5,7 @@ mod snowflake_executor;
 
 mod bash_executor;
 mod bun_executor;
-mod common;
+pub mod common;
 mod config;
 mod dedicated_worker;
 mod deno_executor;

backend/windmill-worker/src/mysql_executor.rs

@@ -26,9 +26,10 @@ pub async fn do_mysql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
 ) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };

backend/windmill-worker/src/pg_executor.rs

@@ -3,11 +3,15 @@ use chrono::Utc;
 use futures::TryStreamExt;
 use native_tls::{Certificate, TlsConnector};
 use postgres_native_tls::MakeTlsConnector;
-use rust_decimal::Decimal;
+use rust_decimal::{prelude::FromPrimitive, Decimal};
 use serde::Deserialize;
 use serde_json::Map;
 use serde_json::{json, Value};
-use tokio_postgres::{types::ToSql, NoTls, Row};
+use tokio_postgres::types::IsNull;
+use tokio_postgres::{
+    types::{to_sql_checked, ToSql},
+    NoTls, Row,
+};
 use tokio_postgres::{
     types::{FromSql, Type},
     Column,
@@ -19,6 +23,7 @@ use windmill_parser_sql::parse_pgsql_sig;
 
 use crate::common::transform_json_value;
 use crate::AuthedClient;
+use bytes::BytesMut;
 use urlencoding::encode;
 
 #[derive(Deserialize)]
@@ -36,9 +41,10 @@ pub async fn do_postgresql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
 ) -> error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };
@@ -127,42 +133,7 @@ pub async fn do_postgresql(
                 .as_ref()
                 .ok_or_else(|| anyhow::anyhow!("Missing otyp for pg arg"))?
                 .to_owned();
-            let boxed: windmill_common::error::Result<Box<dyn ToSql + Sync + Send>> = match value {
-                Value::Null => Ok(Box::new(None::<bool>)),
-                Value::Bool(b) => Ok(Box::new(b.clone())),
-                Value::Number(n) if n.is_i64() && arg_t == "char" => {
-                    Ok(Box::new(n.as_i64().unwrap() as i8))
-                }
-                Value::Number(n)
-                    if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") =>
-                {
-                    Ok(Box::new(n.as_i64().unwrap() as i16))
-                }
-                Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "serial") => {
-                    Ok(Box::new(n.as_i64().unwrap() as i32))
-                }
-                Value::Number(n) if n.is_i64() => Ok(Box::new(n.as_i64().unwrap())),
-                Value::Number(n) if n.is_u64() && arg_t == "oid" => {
-                    Ok(Box::new(n.as_u64().unwrap() as u32))
-                }
-                Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => {
-                    Ok(Box::new(n.as_u64().unwrap() as i64))
-                }
-                Value::Number(n) if n.is_f64() && arg_t == "real" => {
-                    Ok(Box::new(n.as_f64().unwrap() as f32))
-                }
-                Value::Number(n) if n.is_f64() && arg_t == "double" => {
-                    Ok(Box::new(n.as_f64().unwrap()))
-                }
-                Value::Number(n) => Ok(Box::new(n.as_f64().unwrap())),
-                Value::String(s) if arg_t == "uuid" => Ok(Box::new(Uuid::parse_str(s)?)),
-                Value::String(s) => Ok(Box::new(s.clone())),
-                _ => Err(Error::ExecutionErr(format!(
-                    "Unsupported type in query: {:?} and signature {arg_t:?}",
-                    value
-                ))),
-            };
-            boxed
+            convert_val(value, arg_t)
         })
         .collect::<windmill_common::error::Result<Vec<_>>>()?;
     // Now we can execute a simple statement that just returns its parameter.
@@ -184,6 +155,122 @@ pub async fn do_postgresql(
     return Ok(result);
 }
 
+#[derive(Debug)]
+enum PgType {
+    String(String),
+    Bool(bool),
+    I8(i8),
+    I16(i16),
+    I32(i32),
+    I64(i64),
+    U32(u32),
+    F32(f32),
+    F64(f64),
+    Uuid(Uuid),
+    Decimal(Decimal),
+    Date(chrono::NaiveDate),
+    Time(chrono::NaiveTime),
+    Timestamp(chrono::NaiveDateTime),
+    None(Option<bool>),
+    Array(Vec<PgType>),
+}
+
+impl ToSql for PgType {
+    fn to_sql(
+        &self,
+        ty: &Type,
+        out: &mut BytesMut,
+    ) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>> {
+        match *self {
+            PgType::String(ref val) => val.to_sql(ty, out),
+            PgType::Bool(ref val) => val.to_sql(ty, out),
+            PgType::I8(ref val) => val.to_sql(ty, out),
+            PgType::I16(ref val) => val.to_sql(ty, out),
+            PgType::I32(ref val) => val.to_sql(ty, out),
+            PgType::I64(ref val) => val.to_sql(ty, out),
+            PgType::U32(ref val) => val.to_sql(ty, out),
+            PgType::F32(ref val) => val.to_sql(ty, out),
+            PgType::F64(ref val) => val.to_sql(ty, out),
+            PgType::Uuid(ref val) => val.to_sql(ty, out),
+            PgType::Decimal(ref val) => val.to_sql(ty, out),
+            PgType::Date(ref val) => val.to_sql(ty, out),
+            PgType::Time(ref val) => val.to_sql(ty, out),
+            PgType::Timestamp(ref val) => val.to_sql(ty, out),
+            PgType::None(ref val) => val.to_sql(ty, out),
+            PgType::Array(ref val) => val.to_sql(ty, out),
+        }
+    }
+
+    fn accepts(_: &Type) -> bool {
+        true
+    }
+
+    to_sql_checked!();
+}
+
+fn convert_val(value: &Value, arg_t: &String) -> windmill_common::error::Result<PgType> {
+    match value {
+        Value::Array(vec) if arg_t.ends_with("[]") => {
+            let arg_t = arg_t.trim_end_matches("[]").to_string();
+            let mut result = vec![];
+            for val in vec {
+                result.push(convert_val(val, &arg_t)?);
+            }
+            Ok(PgType::Array(result))
+        }
+        Value::Null => Ok(PgType::None(None::<bool>)),
+        Value::Bool(b) => Ok(PgType::Bool(b.clone())),
+        Value::Number(n) if n.is_i64() && arg_t == "char" => {
+            Ok(PgType::I8(n.as_i64().unwrap() as i8))
+        }
+        Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => {
+            Ok(PgType::I16(n.as_i64().unwrap() as i16))
+        }
+        Value::Number(n) if n.is_i64() && (arg_t == "int" || arg_t == "serial") => {
+            Ok(PgType::I32(n.as_i64().unwrap() as i32))
+        }
+        Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
+            PgType::Decimal(Decimal::from_i64(n.as_i64().unwrap()).unwrap()),
+        ),
+        Value::Number(n) if n.is_i64() => Ok(PgType::I64(n.as_i64().unwrap())),
+        Value::Number(n) if n.is_u64() && arg_t == "oid" => {
+            Ok(PgType::U32(n.as_u64().unwrap() as u32))
+        }
+        Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => {
+            Ok(PgType::I64(n.as_u64().unwrap() as i64))
+        }
+        Value::Number(n) if n.is_f64() && arg_t == "real" => {
+            Ok(PgType::F32(n.as_f64().unwrap() as f32))
+        }
+        Value::Number(n) if n.is_f64() && arg_t == "double" => Ok(PgType::F64(n.as_f64().unwrap())),
+        Value::Number(n) if n.is_f64() && (arg_t == "numeric" || arg_t == "decimal") => Ok(
+            PgType::Decimal(Decimal::from_f64(n.as_f64().unwrap()).unwrap()),
+        ),
+        Value::Number(n) => Ok(PgType::F64(n.as_f64().unwrap())),
+        Value::String(s) if arg_t == "uuid" => Ok(PgType::Uuid(Uuid::parse_str(s)?)),
+        Value::String(s) if arg_t == "date" => {
+            let date =
+                chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
+            Ok(PgType::Date(date))
+        }
+        Value::String(s) if arg_t == "time" => {
+            let time =
+                chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default();
+            Ok(PgType::Time(time))
+        }
+        Value::String(s) if arg_t == "timestamp" => {
+            let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ")
+                .unwrap_or_default();
+            Ok(PgType::Timestamp(datetime))
+        }
+        Value::String(s) => Ok(PgType::String(s.clone())),
+        _ => Err(Error::ExecutionErr(format!(
+            "Unsupported type in query: {:?} and signature {arg_t:?}",
+            value
+        ))),
+    }
+}
+
 pub fn pg_cell_to_json_value(
     row: &Row,
     column: &Column,

backend/windmill-worker/src/python_executor.rs

@@ -13,6 +13,7 @@ use windmill_common::{
     error::{self, Error},
     jobs::QueuedJob,
     utils::calculate_hash,
+    worker::WORKER_CONFIG,
 };
 
 lazy_static::lazy_static! {
@@ -25,22 +26,8 @@ lazy_static::lazy_static! {
 
 
     static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
-    static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
     static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
-    static ref PIP_LOCAL_DEPENDENCIES: Option<Vec<String>> = {
-        let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
-            .ok()
-            .map(|x| x.split(',').map(|x| x.to_string()).collect());
-        if pip_local_dependencies == Some(vec!["".to_string()]) {
-            None
-        } else {
-            pip_local_dependencies
-        }
-    };
 
-    static ref ADDITIONAL_PYTHON_PATHS: Option<Vec<String>> = std::env::var("ADDITIONAL_PYTHON_PATHS")
-        .ok()
-        .map(|x| x.split(':').map(|x| x.to_string()).collect());
 
     static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
 
@@ -59,10 +46,10 @@ use crate::S3_CACHE_BUCKET;
 use crate::{
     common::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
-        write_file,
+        start_child_process, write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HTTPS_PROXY, HTTP_PROXY,
-    LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, TZ_ENV,
+    LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, TZ_ENV,
 };
 
 pub async fn create_dependencies_dir(job_dir: &str) {
@@ -85,7 +72,9 @@ pub async fn pip_compile(
     logs.push_str(&format!("\nresolving dependencies..."));
     set_logs(logs, job_id, db).await;
     logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements));
-    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
+    let requirements = if let Some(pip_local_dependencies) =
+        WORKER_CONFIG.read().await.pip_local_dependencies.as_ref()
+    {
         let deps = pip_local_dependencies.clone();
         requirements
             .lines()
@@ -117,7 +106,8 @@ pub async fn pip_compile(
     write_file(job_dir, file, &requirements).await?;
 
     let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
-    if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+    let pip_extra_index_url = PIP_EXTRA_INDEX_URL.read().await.clone();
+    if let Some(url) = pip_extra_index_url.as_ref() {
         args.extend(["--extra-index-url", url]);
     }
     if let Some(url) = PIP_INDEX_URL.as_ref() {
@@ -127,17 +117,18 @@ pub async fn pip_compile(
         args.extend(["--trusted-host", host]);
     }
 
-    let child = Command::new("pip-compile")
+    let mut child_cmd = Command::new("pip-compile");
+    child_cmd
         .current_dir(job_dir)
         .args(args)
         .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()?;
+        .stderr(Stdio::piped());
+    let child_process = start_child_process(child_cmd, "pip-compile").await?;
     handle_child(
         job_id,
         db,
         logs,
-        child,
+        child_process,
         false,
         worker_name,
         &w_id,
@@ -182,8 +173,13 @@ pub async fn handle_python_job(
 ) -> windmill_common::error::Result<serde_json::Value> {
     create_dependencies_dir(job_dir).await;
 
-    let mut additional_python_paths: Vec<String> =
-        ADDITIONAL_PYTHON_PATHS.to_owned().unwrap_or_else(|| vec![]);
+    let mut additional_python_paths: Vec<String> = WORKER_CONFIG
+        .read()
+        .await
+        .additional_python_paths
+        .clone()
+        .unwrap_or_else(|| vec![])
+        .clone();
 
     let requirements = match requirements_o {
         Some(r) => r,
@@ -289,7 +285,7 @@ pub async fn handle_python_job(
         .collect::<Vec<String>>()
         .join("");
     let client = client.get_authed().await;
-    create_args_and_out_file(&client, job, job_dir).await?;
+    create_args_and_out_file(&client, job, job_dir, db).await?;
 
     let import_loader = if relative_imports {
         "import loader"
@@ -434,7 +430,8 @@ mount {{
         job.id
     );
     let child = if !*DISABLE_NSJAIL {
-        Command::new(NSJAIL_PATH.as_str())
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
             .current_dir(job_dir)
             .env_clear()
             // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
@@ -452,10 +449,11 @@ mount {{
                 "wrapper",
             ])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
     } else {
-        Command::new(PYTHON_PATH.as_str())
+        let mut python_cmd = Command::new(PYTHON_PATH.as_str());
+        python_cmd
             .current_dir(job_dir)
             .env_clear()
             .envs(envs)
@@ -465,8 +463,8 @@ mount {{
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(vec!["-u", "-m", "wrapper"])
             .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .spawn()?
+            .stderr(Stdio::piped());
+        start_child_process(python_cmd, PYTHON_PATH.as_str()).await?
     };
 
     handle_child(
@@ -497,8 +495,11 @@ pub async fn handle_python_reqs(
 ) -> error::Result<Vec<String>> {
     let mut req_paths: Vec<String> = vec![];
     let mut vars = vec![("PATH", PATH_ENV.as_str())];
+    let pip_extra_index_url;
+
     if !*DISABLE_NSJAIL {
-        if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+        pip_extra_index_url = PIP_EXTRA_INDEX_URL.read().await.clone();
+        if let Some(url) = pip_extra_index_url.as_ref() {
             vars.push(("EXTRA_INDEX_URL", url));
         }
         if let Some(url) = PIP_INDEX_URL.as_ref() {
@@ -571,14 +572,15 @@ pub async fn handle_python_reqs(
             let req = req.to_string();
             vars.push(("REQ", &req));
             vars.push(("TARGET", &venv_p));
-            Command::new(NSJAIL_PATH.as_str())
+            let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+            nsjail_cmd
                 .current_dir(job_dir)
                 .env_clear()
                 .envs(vars)
                 .args(vec!["--config", "download.config.proto"])
                 .stdout(Stdio::piped())
-                .stderr(Stdio::piped())
-                .spawn()?
+                .stderr(Stdio::piped());
+            start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
         } else {
             let mut command_args = vec![
                 PYTHON_PATH.as_str(),
@@ -595,7 +597,8 @@ pub async fn handle_python_reqs(
                 "-t",
                 venv_p.as_str(),
             ];
-            if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+            let pip_extra_index_url = PIP_EXTRA_INDEX_URL.read().await.clone();
+            if let Some(url) = pip_extra_index_url.as_ref() {
                 command_args.extend(["--extra-index-url", url]);
             }
             if let Some(url) = PIP_INDEX_URL.as_ref() {
@@ -615,7 +618,8 @@ pub async fn handle_python_reqs(
                 envs.push(("NO_PROXY", no_proxy));
             }
 
-            Command::new(FLOCK_PATH.as_str())
+            let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
+            flock_cmd
                 .env_clear()
                 .envs(envs)
                 .args([
@@ -625,8 +629,8 @@ pub async fn handle_python_reqs(
                     &command_args.join(" "),
                 ])
                 .stdout(Stdio::piped())
-                .stderr(Stdio::piped())
-                .spawn()?
+                .stderr(Stdio::piped());
+            start_child_process(flock_cmd, FLOCK_PATH.as_str()).await?
         };
 
         let child = handle_child(

backend/windmill-worker/src/snowflake_executor.rs

@@ -64,9 +64,10 @@ pub async fn do_snowflake(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
 ) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };

backend/windmill-worker/src/worker.rs

@@ -35,7 +35,9 @@ use windmill_common::{
     worker::{update_ping, CLOUD_HOSTED, WORKER_CONFIG},
     DB, IS_READY, METRICS_ENABLED,
 };
-use windmill_queue::{canceled_job_to_result, get_queued_job, pull, HTTP_CLIENT};
+use windmill_queue::{
+    canceled_job_to_result, get_queued_job, pull, push, PushIsolationLevel, HTTP_CLIENT,
+};
 
 use serde_json::{json, Value};
 
@@ -68,7 +70,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error};
 
 use crate::{
     bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE},
-    bun_executor::{gen_lockfile, handle_bun_job},
+    bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job},
     common::{hash_args, read_result, save_in_cache, transform_json_value, write_file},
     deno_executor::{generate_deno_lock, handle_deno_job},
     go_executor::{handle_go_job, install_go_dependencies},
@@ -223,7 +225,8 @@ lazy_static::lazy_static! {
     pub static ref NETRC: Option<String> = std::env::var("NETRC").ok();
 
 
-    pub static ref NPM_CONFIG_REGISTRY: Option<String> = std::env::var("NPM_CONFIG_REGISTRY").ok();
+    pub static ref NPM_CONFIG_REGISTRY: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
+    pub static ref PIP_EXTRA_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
 
 
 
@@ -282,7 +285,6 @@ lazy_static::lazy_static! {
     pub static ref CAN_PULL: Arc<RwLock<()>> = Arc::new(RwLock::new(()));
 
 
-
 }
 //only matter if CLOUD_HOSTED
 pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB
@@ -895,6 +897,13 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     #[cfg(feature = "benchmark")]
     tracing::info!("pre loop time {}s", start.elapsed().as_secs_f64());
 
+    if i_worker == 1 {
+        if let Err(e) =
+            queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name, rsmq.clone()).await
+        {
+            tracing::error!("Error queuing init bash script for worker {worker_name}: {e}");
+        }
+    }
     loop {
         #[cfg(feature = "benchmark")]
         let loop_start = Instant::now();
@@ -1273,6 +1282,52 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     println!("worker {} exited", i_worker);
 }
 
+async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
+    db: &Pool<Postgres>,
+    same_worker_tx: Sender<Uuid>,
+    worker_name: &str,
+    rsmq: Option<R>,
+) -> error::Result<()> {
+    if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() {
+        let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
+        let (uuid, inner_tx) = push(
+            &db,
+            tx,
+            "admins",
+            windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode {
+                content: content.clone(),
+                path: Some(format!("init_script_{worker_name}")),
+                language: ScriptLang::Bash,
+                lock: None,
+                concurrent_limit: None,
+                concurrency_time_window_s: None,
+                cache_ttl: None,
+            }),
+            serde_json::Map::new(),
+            worker_name,
+            "worker@windmill.dev",
+            SUPERADMIN_SECRET_EMAIL.to_string(),
+            None,
+            None,
+            None,
+            None,
+            None,
+            false,
+            true,
+            None,
+            true,
+            None,
+            None,
+            None,
+        )
+        .await?;
+        inner_tx.commit().await?;
+        same_worker_tx.send(uuid).await.map_err(to_anyhow)?;
+        tracing::info!("Creating initial job {uuid} from initial script script: {content}");
+    }
+    Ok(())
+}
+
 // async fn process_result<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
 //     client: AuthedClient,
 //     job: QueuedJob,
@@ -1531,9 +1586,10 @@ async fn do_nativets(
     logs: String,
     client: &AuthedClient,
     code: String,
+    db: &Pool<Postgres>,
 ) -> windmill_common::error::Result<(serde_json::Value, String)> {
     let args = if let Some(args) = &job.args {
-        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone(), &job, db).await?)
     } else {
         None
     };
@@ -1919,9 +1975,9 @@ async fn handle_code_execution_job(
     };
 
     if language == Some(ScriptLang::Postgresql) {
-        return do_postgresql(job.clone(), &client.get_authed().await, &inner_content).await;
+        return do_postgresql(job.clone(), &client.get_authed().await, &inner_content, db).await;
     } else if language == Some(ScriptLang::Mysql) {
-        return do_mysql(job.clone(), &client.get_authed().await, &inner_content).await;
+        return do_mysql(job.clone(), &client.get_authed().await, &inner_content, db).await;
     } else if language == Some(ScriptLang::Bigquery) {
         #[cfg(not(feature = "enterprise"))]
         {
@@ -1932,7 +1988,7 @@ async fn handle_code_execution_job(
 
         #[cfg(feature = "enterprise")]
         {
-            return do_bigquery(job.clone(), &client.get_authed().await, &inner_content).await;
+            return do_bigquery(job.clone(), &client.get_authed().await, &inner_content, db).await;
         }
     } else if language == Some(ScriptLang::Snowflake) {
         #[cfg(not(feature = "enterprise"))]
@@ -1944,10 +2000,10 @@ async fn handle_code_execution_job(
 
         #[cfg(feature = "enterprise")]
         {
-            return do_snowflake(job.clone(), &client.get_authed().await, &inner_content).await;
+            return do_snowflake(job.clone(), &client.get_authed().await, &inner_content, db).await;
         }
     } else if language == Some(ScriptLang::Graphql) {
-        return do_graphql(job.clone(), &client.get_authed().await, &inner_content).await;
+        return do_graphql(job.clone(), &client.get_authed().await, &inner_content, db).await;
     } else if language == Some(ScriptLang::Nativets) {
         logs.push_str("\n--- FETCH TS EXECUTION ---\n");
         let code = format!(
@@ -1955,8 +2011,14 @@ async fn handle_code_execution_job(
             &client.get_token().await,
             inner_content
         );
-        let (result, ts_logs) =
-            do_nativets(job.clone(), logs.clone(), &client.get_authed().await, code).await?;
+        let (result, ts_logs) = do_nativets(
+            job.clone(),
+            logs.clone(),
+            &client.get_authed().await,
+            code,
+            db,
+        )
+        .await?;
         *logs = ts_logs;
         return Ok(result);
     }
@@ -2275,7 +2337,7 @@ async fn lock_modules(
                             db,
                             worker_name,
                             worker_dir,
-                            job_path.clone(),
+                            job_path,
                             base_internal_url,
                             token,
                         )
@@ -2296,7 +2358,7 @@ async fn lock_modules(
                             db,
                             worker_name,
                             worker_dir,
-                            job_path.clone(),
+                            job_path,
                             base_internal_url,
                             token,
                         )
@@ -2316,7 +2378,7 @@ async fn lock_modules(
                             db,
                             worker_name,
                             worker_dir,
-                            job_path.clone(),
+                            job_path,
                             base_internal_url,
                             token,
                         )
@@ -2331,7 +2393,7 @@ async fn lock_modules(
                         db,
                         worker_name,
                         worker_dir,
-                        job_path.clone(),
+                        job_path,
                         base_internal_url,
                         token,
                     )
@@ -2671,6 +2733,8 @@ async fn capture_dependency_job(
         }
         ScriptLang::Bun => {
             let _ = write_file(job_dir, "main.ts", job_raw_code).await?;
+            //TODO: remove once bun provides sane default fot it
+            let trusted_deps = get_trusted_deps(job_raw_code);
             let req = gen_lockfile(
                 logs,
                 job_id,
@@ -2682,6 +2746,7 @@ async fn capture_dependency_job(
                 base_internal_url,
                 worker_name,
                 true,
+                trusted_deps,
             )
             .await?;
             Ok(req.unwrap_or_else(String::new))

backend/windmill-worker/src/worker_flow.rs

@@ -70,6 +70,7 @@ pub async fn update_flow_status_after_job_completion<
         same_worker_tx.clone(),
         worker_dir,
         stop_early_override,
+        false,
         rsmq.clone(),
     )
     .await?;
@@ -87,6 +88,7 @@ pub async fn update_flow_status_after_job_completion<
             same_worker_tx.clone(),
             worker_dir,
             nrec.stop_early_override,
+            nrec.skip_error_handler,
             rsmq.clone(),
         )
         .await?;
@@ -99,6 +101,7 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
     success: bool,
     result: serde_json::Value,
     stop_early_override: Option<bool>,
+    skip_error_handler: bool,
 }
 // #[instrument(level = "trace", skip_all)]
 pub async fn update_flow_status_after_job_completion_internal<
@@ -116,9 +119,10 @@ pub async fn update_flow_status_after_job_completion_internal<
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     stop_early_override: Option<bool>,
+    skip_error_handler: bool,
     rsmq: Option<R>,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
-    let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult) = {
+    let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult, is_failure_step) = {
         // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
 
         let old_status_json = sqlx::query_scalar!(
@@ -162,10 +166,23 @@ pub async fn update_flow_status_after_job_completion_internal<
             (false, None)
         };
 
-        let is_failure_step = old_status.step >= old_status.modules.len() as i32;
+        // 0 length flows are not failure steps
+        let is_failure_step =
+            old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0;
 
         let (mut stop_early, skip_if_stop_early) = if let Some(se) = stop_early_override {
-            (true, se)
+            //do not stop early if module is a flow step
+            let mut tx = db.begin().await?;
+            let flow_job = get_queued_job(flow, w_id, &mut tx)
+                .await?
+                .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
+            tx.commit().await?;
+            let module = get_module(&flow_job, module_index);
+            if module.is_some_and(|x| matches!(x.value, FlowModuleValue::Flow { .. })) {
+                (false, false)
+            } else {
+                (true, se)
+            }
         } else if is_failure_step {
             (false, false)
         } else {
@@ -204,8 +221,6 @@ pub async fn update_flow_status_after_job_completion_internal<
             _ => false,
         };
 
-        let skip_failure = skip_branch_failure || skip_loop_failures;
-
         let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
 
         let (inc_step_counter, new_status) = match module_status {
@@ -327,6 +342,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                     _ => (None, None),
                 };
                 if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
+                    success = true;
                     (
                         true,
                         Some(FlowStatusModule::Success {
@@ -485,22 +501,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             .unwrap_or_else(|| "none".to_string());
         tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
 
-        let module = {
-            let raw_flow = flow_job.parse_raw_flow();
-            if let Some(raw_flow) = raw_flow {
-                if let Some(i) = module_index {
-                    if let Some(module) = raw_flow.modules.get(i) {
-                        Some(module.clone())
-                    } else {
-                        raw_flow.failure_module
-                    }
-                } else {
-                    None
-                }
-            } else {
-                None
-            }
-        };
+        let module = get_module(&flow_job, module_index);
 
         // tracing::error!(
         //     "UPDATE FLOW STATUS 3: {module:#?} {skip_failure} {is_last_step} {success}"
@@ -510,7 +511,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             _ if flow_job.canceled => false,
             true => !is_last_step,
             false if unrecoverable => false,
-            false if skip_failure => !is_last_step,
+            false if skip_branch_failure || skip_loop_failures => !is_last_step,
             false
                 if next_retry(
                     &module.and_then(|m| m.retry.clone()).unwrap_or_default(),
@@ -520,7 +521,11 @@ pub async fn update_flow_status_after_job_completion_internal<
             {
                 true
             }
-            false if has_failure_module(flow, tx.transaction_mut()).await? && !is_failure_step => {
+            false
+                if !is_failure_step
+                    && !skip_error_handler
+                    && has_failure_module(flow, tx.transaction_mut()).await? =>
+            {
                 true
             }
             false => false,
@@ -548,6 +553,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             stop_early,
             skip_if_stop_early,
             nresult,
+            is_failure_step,
         )
     };
 
@@ -584,7 +590,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             add_completed_job(
                 db,
                 &flow_job,
-                success,
+                success && !is_failure_step && !skip_error_handler,
                 stop_early && skip_if_stop_early,
                 &nresult,
                 logs,
@@ -631,13 +637,14 @@ pub async fn update_flow_status_after_job_completion_internal<
             return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
                 flow: parent_job,
                 job_id_for_status: flow,
-                success,
+                success: success && !is_failure_step,
                 result: nresult,
                 stop_early_override: if stop_early {
                     Some(skip_if_stop_early)
                 } else {
                     None
                 },
+                skip_error_handler: skip_error_handler || is_failure_step,
             }));
         }
         Ok(None)
@@ -646,6 +653,23 @@ pub async fn update_flow_status_after_job_completion_internal<
     }
 }
 
+fn get_module(flow_job: &QueuedJob, module_index: Option<usize>) -> Option<FlowModule> {
+    let raw_flow = flow_job.parse_raw_flow();
+    if let Some(raw_flow) = raw_flow {
+        if let Some(i) = module_index {
+            if let Some(module) = raw_flow.modules.get(i) {
+                Some(module.clone())
+            } else {
+                raw_flow.failure_module
+            }
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
+
 async fn compute_skip_loop_failures_and_parallelism(
     flow: Uuid,
     step: i32,
@@ -733,7 +757,7 @@ async fn compute_bool_from_expr(
             "resume".to_string(),
             resumes.0.last().map(|v| json!(v)).unwrap_or_default(),
         ));
-        env.push(("resumes".to_string(), resumes.0.clone().into()));
+        env.push(("resumes".to_string(), resumes.0.into()));
         env.push(("approvers".to_string(), json!(resumes.1.clone())));
     }
 
@@ -844,7 +868,7 @@ async fn transform_input(
                         "resume".to_string(),
                         resumes.last().map(|v| json!(v)).unwrap_or_default(),
                     ),
-                    ("resumes".to_string(), resumes.clone().into()),
+                    ("resumes".to_string(), resumes.into()),
                     ("approvers".to_string(), json!(approvers.clone())),
                 ];
 
@@ -1830,7 +1854,7 @@ async fn compute_next_flow_transform(
                                     "resume".to_string(),
                                     resumes.last().map(|v| json!(v)).unwrap_or_default(),
                                 ),
-                                ("resumes".to_string(), resumes.clone().into()),
+                                ("resumes".to_string(), resumes.into()),
                                 ("approvers".to_string(), json!(approvers.clone())),
                             ]
                         },

benchmarks/benchmark_oneoff.ts

@@ -119,7 +119,7 @@ export async function main({
     return completedJobs - pastJobs;
   }
 
-  if (["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)) {
+  if (["deno", "python", "go", "bash", "dedicated", "bun", "nativets"].includes(kind)) {
     await createBenchScript(kind, workspace);
   }
 
@@ -135,7 +135,7 @@ export async function main({
       kind: "noop",
     });
   } else if (
-    ["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)
+    ["deno", "python", "go", "bash", "dedicated", "bun", "nativets"].includes(kind)
   ) {
     body = JSON.stringify({
       kind: "script",
@@ -234,7 +234,7 @@ export async function main({
   console.log("completed jobs", completedJobs);
   console.log("queue length:", await getQueueCount());
 
-  if (!noVerify && kind !== "noop") {
+  if (!noVerify && kind !== "noop" && kind !== 'nativets') {
     await verifyOutputs(uuids, config.workspace_id);
   }
 
@@ -282,7 +282,7 @@ if (import.meta.main) {
     )
     .option(
       "--kind <kind:string>",
-      "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps",
+      "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets",
       {
         required: true,
       }

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.177.0";
+export const VERSION = "v1.183.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({
@@ -88,7 +88,11 @@ export async function createBenchScript(
     scriptContent =
       'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
     language = "deno";
-  } else {
+  }  else if (scriptPattern === "nativets") {
+    scriptContent =
+      'export async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }';
+    language = "nativets";
+  }  else {
     throw new Error(
       "Could not create script for script pattern " + scriptPattern
     );

benchmarks/suite_config.json

@@ -39,6 +39,11 @@
       "graph_title": "bash throughput benchmark (single worker)",
       "kind": "bash",
       "jobs": 500
+    },
+    {
+      "graph_title": "nativets throughput benchmark (single worker)",
+      "kind": "nativets",
+      "jobs": 500
     }
   ],
   "extra_graphs": [

cli/main.ts

@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
   }
 });
 
-export const VERSION = "v1.177.0";
+export const VERSION = "v1.183.0";
 
 let command: any = new Command()
   .name("wmill")

docker-compose.yml

@@ -35,8 +35,6 @@ services:
       - NUM_WORKERS=0
       - DISABLE_SERVER=false
       - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
-      # LICENSE_KEY is only needed for the enterprise edition
-      - LICENSE_KEY=${WM_LICENSE_KEY}
     depends_on:
       db:
         condition: service_healthy
@@ -58,8 +56,6 @@ services:
       - KEEP_JOB_DIR=false
       - METRICS_ADDR=false
       - WORKER_GROUP=default
-      # LICENSE_KEY is only needed for the enterprise edition
-      - LICENSE_KEY=${WM_LICENSE_KEY}
     depends_on:
       db:
         condition: service_healthy

frontend/.nvmrc

@@ -1 +0,0 @@
-v16.15.0

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.177.0",
+	"version": "1.183.0",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.177.0",
+			"version": "1.183.0",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",
@@ -16,6 +16,8 @@
 				"@popperjs/core": "^2.11.6",
 				"@redocly/json-to-json-schema": "^0.0.1",
 				"@tanstack/svelte-table": "^8.9.9",
+				"ag-grid-community": "^30.2.0",
+				"ag-grid-enterprise": "^30.2.0",
 				"ag-grid-svelte": "^0.3.0",
 				"ansi_up": "^5.2.1",
 				"chart.js": "^4.3.0",
@@ -24,6 +26,7 @@
 				"d3-zoom": "^3.0.0",
 				"date-fns": "^2.30.0",
 				"diff": "^5.1.0",
+				"driver.js": "^1.3.0",
 				"esm-env": "^1.0.0",
 				"fast-equals": "^5.0.1",
 				"graphql": "^16.7.1",
@@ -41,11 +44,10 @@
 				"svelte-exmarkdown": "^2.1.0",
 				"svelte-infinite-loading": "^1.3.8",
 				"svelte-portal": "^2.2.0",
-				"svelte-timezone-picker": "^2.0.3",
 				"svelte-tiny-virtual-list": "^2.0.5",
 				"tailwind-merge": "^1.13.2",
 				"vscode-ws-jsonrpc": "3.0.0",
-				"windmill-parser-wasm": "^1.154.3",
+				"windmill-parser-wasm": "^1.180.0",
 				"y-monaco": "^0.1.4",
 				"y-websocket": "^1.5.0",
 				"yjs": "^13.6.7"
@@ -95,13 +97,13 @@
 				"svelte-overlay": "^1.4.1",
 				"svelte-popperjs": "^1.3.2",
 				"svelte-preprocess": "^5.0.1",
-				"svelte-range-slider-pips": "^2.1.1",
+				"svelte-range-slider-pips": "^2.2.3",
 				"svelte-splitpanes": "^0.8.0",
 				"svelte2tsx": "^0.6.16",
 				"tailwindcss": "^3.3.2",
 				"tslib": "^2.6.1",
 				"typescript": "^5.1.3",
-				"vite": "^4.4.9",
+				"vite": "^4.4.11",
 				"vite-plugin-monaco-editor": "^1.1.0",
 				"yootils": "^0.3.1"
 			},
@@ -119,14 +121,6 @@
 			"version": "1.109.1",
 			"extraneous": true
 		},
-		"../backend/parsers/windmill-parser-wasm/pkg": {
-			"name": "windmill-parser-wasm",
-			"version": "1.127.0",
-			"extraneous": true
-		},
-		"../backendd/parsers/windmill-parser-wasm/pkg": {
-			"extraneous": true
-		},
 		"node_modules/@aashutoshrathi/word-wrap": {
 			"version": "1.2.6",
 			"resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
@@ -2127,10 +2121,14 @@
 			}
 		},
 		"node_modules/ag-grid-community": {
-			"version": "28.2.1",
-			"resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-28.2.1.tgz",
-			"integrity": "sha512-DMZh/xD/FqYP17qJ1M92PolTYe+hrKuEaf+A4h13O6qn2x/xZQrTRGW5DgnQLR/uLMe1XXZQPKR3UKgAlKo69A==",
-			"peer": true
+			"version": "30.2.0",
+			"resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-30.2.0.tgz",
+			"integrity": "sha512-Gd6GXmtzEQSCDloBdRxxCDqnjTBRAOf/zzlaxxyyVBJgc+cePuNgGdplRUhT/rwIiDwvyuoynvxelVE/iYdXsA=="
+		},
+		"node_modules/ag-grid-enterprise": {
+			"version": "30.2.0",
+			"resolved": "https://registry.npmjs.org/ag-grid-enterprise/-/ag-grid-enterprise-30.2.0.tgz",
+			"integrity": "sha512-rOP52n8NyA4doBBYj9crIcYd5vN62nTOOB9552l9hNPDX9oKy8I6jGqXUIoGuQDg4oLkQIoddZ+vv0W+Mf9kYg=="
 		},
 		"node_modules/ag-grid-svelte": {
 			"version": "0.3.0",
@@ -2797,11 +2795,6 @@
 				"node": ">=16"
 			}
 		},
-		"node_modules/compute-scroll-into-view": {
-			"version": "1.0.20",
-			"resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-1.0.20.tgz",
-			"integrity": "sha512-UCB0ioiyj8CRjtrvaceBLqqhZCVP+1B8+NWQhmdsm0VXOJtobBCf1dBQmebCCo34qZmUwZfIH2MZLqNHazrfjg=="
-		},
 		"node_modules/concat-map": {
 			"version": "0.0.1",
 			"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3508,6 +3501,11 @@
 				"url": "https://github.com/fb55/domutils?sponsor=1"
 			}
 		},
+		"node_modules/driver.js": {
+			"version": "1.3.0",
+			"resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.0.tgz",
+			"integrity": "sha512-ilUkVc5iMIYfMd8FdWy8n5Wv//gsJuRP+lo8QfWpwP9c0UGOgD7P9nVQMZwcdW84aqAZHHUHrV7GgiopAN6HUQ=="
+		},
 		"node_modules/earcut": {
 			"version": "2.2.4",
 			"resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz",
@@ -9073,9 +9071,9 @@
 			}
 		},
 		"node_modules/svelte-range-slider-pips": {
-			"version": "2.1.1",
-			"resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.1.1.tgz",
-			"integrity": "sha512-fLkhfnyEc4uMOCZCl8IPyC0e9lujrjqdSyrKw28Y8j99YUPZR+pHnSxde7PqhM0ST6eyO4ugh2nFnLZTIoG9aQ==",
+			"version": "2.2.3",
+			"resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.2.3.tgz",
+			"integrity": "sha512-ZAwX9+NDOE6cdqUTMfwBw27YI7NFgSXyPS1ARx3Zuaish2q6FuAtgx5d9+uqNzxQ7WVFhxgM7iQp8MvCpvy4Jw==",
 			"dev": true
 		},
 		"node_modules/svelte-splitpanes": {
@@ -9090,21 +9088,6 @@
 				"svelte": "^4.0.2"
 			}
 		},
-		"node_modules/svelte-timezone-picker": {
-			"version": "2.0.3",
-			"resolved": "https://registry.npmjs.org/svelte-timezone-picker/-/svelte-timezone-picker-2.0.3.tgz",
-			"integrity": "sha512-p2gqAwuIeDGAxxTI21A4VPY4QrDaobuCq5OSCVFbjvS9Ok9JvVv6vQRR2i+E2TMx6AytXV9CGs1dnpvE/lwvHQ==",
-			"dependencies": {
-				"compute-scroll-into-view": "^1.0.16"
-			},
-			"engines": {
-				"node": ">= 10",
-				"npm": ">= 6"
-			},
-			"peerDependencies": {
-				"svelte": "^3.25.0"
-			}
-		},
 		"node_modules/svelte-tiny-virtual-list": {
 			"version": "2.0.5",
 			"resolved": "https://registry.npmjs.org/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-2.0.5.tgz",
@@ -9748,9 +9731,9 @@
 			}
 		},
 		"node_modules/vite": {
-			"version": "4.4.9",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz",
-			"integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==",
+			"version": "4.4.11",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
+			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
 			"dev": true,
 			"dependencies": {
 				"esbuild": "^0.18.10",
@@ -9972,9 +9955,9 @@
 			}
 		},
 		"node_modules/windmill-parser-wasm": {
-			"version": "1.154.3",
-			"resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.154.3.tgz",
-			"integrity": "sha512-mrp9HFhxVzNJRGr7L8Am/JawQkoTsQi9Ht3TkT9b+VzSEzSC5wZD1mukPFBK/oEQK+NReDcvNCg5LTSPv1caAQ=="
+			"version": "1.180.0",
+			"resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.180.0.tgz",
+			"integrity": "sha512-ikkyfiAdILGUZT0g+eiZ9dgPYFHCjtZVFCRUAkw7PlbK+8bT6ROa+mv1ZRxkc3LZLGsWEwteojdcjfopPFTTAQ=="
 		},
 		"node_modules/wordwrap": {
 			"version": "1.0.0",
@@ -11564,10 +11547,14 @@
 			"requires": {}
 		},
 		"ag-grid-community": {
-			"version": "28.2.1",
-			"resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-28.2.1.tgz",
-			"integrity": "sha512-DMZh/xD/FqYP17qJ1M92PolTYe+hrKuEaf+A4h13O6qn2x/xZQrTRGW5DgnQLR/uLMe1XXZQPKR3UKgAlKo69A==",
-			"peer": true
+			"version": "30.2.0",
+			"resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-30.2.0.tgz",
+			"integrity": "sha512-Gd6GXmtzEQSCDloBdRxxCDqnjTBRAOf/zzlaxxyyVBJgc+cePuNgGdplRUhT/rwIiDwvyuoynvxelVE/iYdXsA=="
+		},
+		"ag-grid-enterprise": {
+			"version": "30.2.0",
+			"resolved": "https://registry.npmjs.org/ag-grid-enterprise/-/ag-grid-enterprise-30.2.0.tgz",
+			"integrity": "sha512-rOP52n8NyA4doBBYj9crIcYd5vN62nTOOB9552l9hNPDX9oKy8I6jGqXUIoGuQDg4oLkQIoddZ+vv0W+Mf9kYg=="
 		},
 		"ag-grid-svelte": {
 			"version": "0.3.0",
@@ -12028,11 +12015,6 @@
 			"integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==",
 			"dev": true
 		},
-		"compute-scroll-into-view": {
-			"version": "1.0.20",
-			"resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-1.0.20.tgz",
-			"integrity": "sha512-UCB0ioiyj8CRjtrvaceBLqqhZCVP+1B8+NWQhmdsm0VXOJtobBCf1dBQmebCCo34qZmUwZfIH2MZLqNHazrfjg=="
-		},
 		"concat-map": {
 			"version": "0.0.1",
 			"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -12543,6 +12525,11 @@
 				"domhandler": "^5.0.3"
 			}
 		},
+		"driver.js": {
+			"version": "1.3.0",
+			"resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.0.tgz",
+			"integrity": "sha512-ilUkVc5iMIYfMd8FdWy8n5Wv//gsJuRP+lo8QfWpwP9c0UGOgD7P9nVQMZwcdW84aqAZHHUHrV7GgiopAN6HUQ=="
+		},
 		"earcut": {
 			"version": "2.2.4",
 			"resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz",
@@ -16433,9 +16420,9 @@
 			}
 		},
 		"svelte-range-slider-pips": {
-			"version": "2.1.1",
-			"resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.1.1.tgz",
-			"integrity": "sha512-fLkhfnyEc4uMOCZCl8IPyC0e9lujrjqdSyrKw28Y8j99YUPZR+pHnSxde7PqhM0ST6eyO4ugh2nFnLZTIoG9aQ==",
+			"version": "2.2.3",
+			"resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.2.3.tgz",
+			"integrity": "sha512-ZAwX9+NDOE6cdqUTMfwBw27YI7NFgSXyPS1ARx3Zuaish2q6FuAtgx5d9+uqNzxQ7WVFhxgM7iQp8MvCpvy4Jw==",
 			"dev": true
 		},
 		"svelte-splitpanes": {
@@ -16447,14 +16434,6 @@
 				"esm-env-robust": "0.0.3"
 			}
 		},
-		"svelte-timezone-picker": {
-			"version": "2.0.3",
-			"resolved": "https://registry.npmjs.org/svelte-timezone-picker/-/svelte-timezone-picker-2.0.3.tgz",
-			"integrity": "sha512-p2gqAwuIeDGAxxTI21A4VPY4QrDaobuCq5OSCVFbjvS9Ok9JvVv6vQRR2i+E2TMx6AytXV9CGs1dnpvE/lwvHQ==",
-			"requires": {
-				"compute-scroll-into-view": "^1.0.16"
-			}
-		},
 		"svelte-tiny-virtual-list": {
 			"version": "2.0.5",
 			"resolved": "https://registry.npmjs.org/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-2.0.5.tgz",
@@ -16907,9 +16886,9 @@
 			}
 		},
 		"vite": {
-			"version": "4.4.9",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz",
-			"integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==",
+			"version": "4.4.11",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
+			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
 			"dev": true,
 			"requires": {
 				"esbuild": "^0.18.10",
@@ -17050,9 +17029,9 @@
 			}
 		},
 		"windmill-parser-wasm": {
-			"version": "1.154.3",
-			"resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.154.3.tgz",
-			"integrity": "sha512-mrp9HFhxVzNJRGr7L8Am/JawQkoTsQi9Ht3TkT9b+VzSEzSC5wZD1mukPFBK/oEQK+NReDcvNCg5LTSPv1caAQ=="
+			"version": "1.180.0",
+			"resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.180.0.tgz",
+			"integrity": "sha512-ikkyfiAdILGUZT0g+eiZ9dgPYFHCjtZVFCRUAkw7PlbK+8bT6ROa+mv1ZRxkc3LZLGsWEwteojdcjfopPFTTAQ=="
 		},
 		"wordwrap": {
 			"version": "1.0.0",

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.177.0",
+	"version": "1.183.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -60,13 +60,13 @@
 		"svelte-overlay": "^1.4.1",
 		"svelte-popperjs": "^1.3.2",
 		"svelte-preprocess": "^5.0.1",
-		"svelte-range-slider-pips": "^2.1.1",
+		"svelte-range-slider-pips": "^2.2.3",
 		"svelte-splitpanes": "^0.8.0",
 		"svelte2tsx": "^0.6.16",
 		"tailwindcss": "^3.3.2",
 		"tslib": "^2.6.1",
 		"typescript": "^5.1.3",
-		"vite": "^4.4.9",
+		"vite": "^4.4.11",
 		"vite-plugin-monaco-editor": "^1.1.0",
 		"yootils": "^0.3.1"
 	},
@@ -93,6 +93,8 @@
 		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
 		"@tanstack/svelte-table": "^8.9.9",
+		"ag-grid-community": "^30.2.0",
+		"ag-grid-enterprise": "^30.2.0",
 		"ag-grid-svelte": "^0.3.0",
 		"ansi_up": "^5.2.1",
 		"chart.js": "^4.3.0",
@@ -101,6 +103,7 @@
 		"d3-zoom": "^3.0.0",
 		"date-fns": "^2.30.0",
 		"diff": "^5.1.0",
+		"driver.js": "^1.3.0",
 		"esm-env": "^1.0.0",
 		"fast-equals": "^5.0.1",
 		"graphql": "^16.7.1",
@@ -118,11 +121,10 @@
 		"svelte-exmarkdown": "^2.1.0",
 		"svelte-infinite-loading": "^1.3.8",
 		"svelte-portal": "^2.2.0",
-		"svelte-timezone-picker": "^2.0.3",
 		"svelte-tiny-virtual-list": "^2.0.5",
 		"tailwind-merge": "^1.13.2",
 		"vscode-ws-jsonrpc": "3.0.0",
-		"windmill-parser-wasm": "^1.154.3",
+		"windmill-parser-wasm": "^1.180.0",
 		"y-monaco": "^0.1.4",
 		"y-websocket": "^1.5.0",
 		"yjs": "^13.6.7"

frontend/src/lib/common.ts

@@ -36,6 +36,7 @@ export type Schema = {
 	$schema: string | undefined
 	type: string
 	properties: { [name: string]: SchemaProperty }
+	order?: string[]
 	required: string[]
 }
 

frontend/src/lib/components/ArgInput.svelte

@@ -289,7 +289,7 @@
 									{#if i < itemsLimit}
 										<div class="flex max-w-md mt-1 w-full items-center">
 											{#if itemsType?.type == 'number'}
-												<input type="number" bind:value={v} />
+												<input type="number" bind:value={v} id="arg-input-number-array" />
 											{:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'}
 												<input
 													type="file"
@@ -348,6 +348,7 @@
 								}
 								value = value.concat('')
 							}}
+							id="arg-input-add-item"
 						>
 							<Icon data={faPlus} class="mr-2" />
 							Add item

frontend/src/lib/components/ArrayTypeNarrowing.svelte

@@ -34,6 +34,7 @@
 			itemsType = undefined
 		}
 	}}
+	id="array-type-narrowing"
 >
 	<option value="string"> Items are strings</option>
 	<option value="enum">Items are strings from an enum</option>

frontend/src/lib/components/CenteredPage.svelte

@@ -3,7 +3,7 @@
 </script>
 
 <div class="pb-8 h-fit-content">
-	<div class={twMerge('max-w-6xl mx-auto px-4 sm:px-6 md:px-8 h-fit-content', $$restProps.class)}>
+	<div class={twMerge('max-w-7xl mx-auto px-4 sm:px-6 md:px-8 h-fit-content', $$restProps.class)}>
 		<slot />
 	</div>
 </div>