feat(frontend): wip

* fix(frontend): fix modifier text size

* fix(frontend): fix modifier text size

.env

@@ -1,19 +1,9 @@
-DB_PASSWORD=changeme
 DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
 
+# For Enterprise Edition, use:
+# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
 WM_IMAGE=ghcr.io/windmill-labs/windmill:main
-WM_LICENSE_KEY=""
 
-# For Enterprise Edition, comment the 2 lines above and uncomment below
-# WINDMILL_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
-# WM_LICENSE_KEY="<id>.<expiry>.<signature>"
-
-
-# this is the url that your instance is publicly exposed to
-WM_BASE_URL=http://localhost
 
 # To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
 # To have caddy take care of automatic TLS
-
-
-WM_REQUEST_SIZE_LIMIT=50097152 # 50MB (The size limit for any requests, including script/flow args inputs)

.github/DockerfileBackendTests

@@ -37,9 +37,10 @@ RUN apt-get update \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr nodejs npm\
+    libv8-dev tesseract-ocr nodejs npm clang libclang-dev\
     && rm -rf /var/lib/apt/lists/*
 
+
 RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
@@ -57,7 +58,7 @@ RUN /usr/local/bin/python3 -m pip install nltk
 RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
+COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
 RUN apt-get update \

.github/change-versions.sh

@@ -5,7 +5,7 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json

.github/workflows/automerge-dependabot.yml

@@ -1,26 +0,0 @@
-name: dependabot auto-merge
-
-on: pull_request_target
-
-permissions:
-  contents: read
-  pull-requests: read
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.6.0
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Enable auto-merge for Dependabot PRs
-        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
-        run: |
-          echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
-          gh pr review --approve "$PR_URL"
-          gh pr merge --auto --squash "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/automerge-dependabot.yml.archived

@@ -0,0 +1,26 @@
+# name: dependabot auto-merge
+
+# on: pull_request_target
+
+# permissions:
+#   contents: read
+#   pull-requests: read
+
+# jobs:
+#   dependabot:
+#     runs-on: ubuntu-latest
+#     if: ${{ github.actor == 'dependabot[bot]' }}
+#     steps:
+#       - name: Dependabot metadata
+#         id: metadata
+#         uses: dependabot/fetch-metadata@v1.6.0
+#         with:
+#           github-token: "${{ secrets.GITHUB_TOKEN }}"
+#       - name: Enable auto-merge for Dependabot PRs
+#         if: steps.metadata.outputs.update-type == 'version-update:semver-patch' ||  steps.metadata.outputs.update-type == 'version-update:semver-minor'
+#         run: |
+#           echo ${{ secrets.RUBEN_PAT }} | gh auth login --with-token
+#           gh pr review --approve "$PR_URL"
+#           gh pr merge --auto --squash "$PR_URL"
+#         env:
+#           PR_URL: ${{github.event.pull_request.html_url}}

.github/workflows/backend-test.yml

@@ -42,4 +42,4 @@ jobs:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
           DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
-          test --all -- --nocapture
+          test --features enterprise --all -- --nocapture 

.github/workflows/benchmark.yml

@@ -19,23 +19,45 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
       windmill:
-        image: ghcr.io/windmill-labs/windmill:main
+        image: ghcr.io/windmill-labs/windmill-ee:main
         env:
           DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow
         options: >-
           --pull always --health-interval 10s --health-timeout 5s
           --health-retries 5 --health-cmd "curl
           http://localhost:8000/api/version"
         ports:
           - 8000:8000
+      windmill-worker:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          DISABLE_SERVER: true
+          WORKER_GROUP: dedicated
+          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+        options: >-
+          --pull always
     steps:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
+      - uses: actions/checkout@v4
+        with:
+          ref: benchmarks
       - name: benchmark
         timeout-minutes: 10
-        run:
-          deno run --unstable -A
-          https://raw.githubusercontent.com/windmill-labs/windmill/main/benchmarks/benchmark_noop.ts
-          --host http://localhost:8000 -e admin@windmill.dev -p changeme -j
-          10000
+        run: deno run --unstable -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
+      - name: Push changes
+        run: |
+          pwd
+          git add .
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git commit -m "Update benchmarks"
+          git push

.github/workflows/docker-image.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
-  contents: read
+  contents: write
   id-token: write
   packages: write
 
@@ -96,6 +96,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
             features=enterprise
@@ -152,7 +153,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 18
-      - uses: shrink/actions-docker-extract@v2
+      - uses: shrink/actions-docker-extract@v3
         id: extract
         with:
           image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
@@ -164,6 +165,97 @@ jobs:
           bucket: windmill-frontend
           bucket-region: us-east-1
 
+
+  attach_amd64_binary_to_release:
+    needs: [build, build_ee]
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    env:
+      ARCH: amd64
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: |
+          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - run: |
+          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
+          # Also useful to visually check that the arch is the right opencontainers
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          path: "/usr/src/app/windmill"
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-ee
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/usr/src/app/windmill"
+
+      - name: Rename binary with corresponding architecture
+        run: |
+          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
+          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
+
+
+      - name: Attach binary to release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{ steps.extract.outputs.destination }}/*
+            ${{ steps.extract-ee.outputs.destination }}/*
+
+  attach_arm64_binary_to_release:
+    needs: [build, build_ee]
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    env:
+      ARCH: arm64
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: |
+          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - run: |
+          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
+          # Also useful to visually check that the arch is the right opencontainers
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          path: "/usr/src/app/windmill"
+
+      - uses: shrink/actions-docker-extract@v3
+        id: extract-ee
+        with:
+          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
+          path: "/usr/src/app/windmill"
+
+      - name: Rename binary with corresponding architecture
+        run: |
+          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
+          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
+
+
+      - name: Attach binary to release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{ steps.extract.outputs.destination }}/*
+            ${{ steps.extract-ee.outputs.destination }}/*
+
   publish_ecr:
     needs: [build_ee]
     runs-on: ubuntu-latest

.github/workflows/sign-cla.yml

@@ -12,7 +12,7 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: cla-assistant/github-action@v2.2.1
+        uses: cla-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}

CHANGELOG.md

@@ -1,6 +1,316 @@
 # Changelog
 
 
+## [1.183.0](https://github.com/windmill-labs/windmill/compare/v1.182.3...v1.183.0) (2023-10-11)
+
+
+### Features
+
+* **frontend:** Table wizard ([#2416](https://github.com/windmill-labs/windmill/issues/2416)) ([6f0cda0](https://github.com/windmill-labs/windmill/commit/6f0cda0e1ea84e2b5c5d297c841749dc5bae879d))
+
+
+### Bug Fixes
+
+* benchmark config syntax error ([#2432](https://github.com/windmill-labs/windmill/issues/2432)) ([109c2f1](https://github.com/windmill-labs/windmill/commit/109c2f17d68e0cac2f365297cc2fcdd54d9d105a))
+* **frontend:** add a validation for base url ([#2434](https://github.com/windmill-labs/windmill/issues/2434)) ([c914ac6](https://github.com/windmill-labs/windmill/commit/c914ac64cfbaacaf5fe3c7486ea9901ce4828387))
+* **frontend:** fix drawer title truncate ([#2429](https://github.com/windmill-labs/windmill/issues/2429)) ([46d2c13](https://github.com/windmill-labs/windmill/commit/46d2c13e0d2dde1e87c3bbe7cc2be29de84fa2cf))
+* **frontend:** fix mobile multi select ([#2431](https://github.com/windmill-labs/windmill/issues/2431)) ([cb2b6df](https://github.com/windmill-labs/windmill/commit/cb2b6dfdba8953a3d1f432e4af2b2725f5e267ca))
+* **frontend:** fix table wizards for old apps ([#2435](https://github.com/windmill-labs/windmill/issues/2435)) ([e088ec5](https://github.com/windmill-labs/windmill/commit/e088ec566958079e468b3c1f5df057f6e70dffc3))
+
+## [1.182.3](https://github.com/windmill-labs/windmill/compare/v1.182.2...v1.182.3) (2023-10-10)
+
+
+### Bug Fixes
+
+* improve binary build ([094539f](https://github.com/windmill-labs/windmill/commit/094539ff3aa79531953f82941337bdd3d34db630))
+
+## [1.182.2](https://github.com/windmill-labs/windmill/compare/v1.182.1...v1.182.2) (2023-10-10)
+
+
+### Bug Fixes
+
+* add binaries to release ([17b42e6](https://github.com/windmill-labs/windmill/commit/17b42e6a3555ae1f45d8f24934f290a72e3d60c5))
+
+## [1.182.1](https://github.com/windmill-labs/windmill/compare/v1.182.0...v1.182.1) (2023-10-10)
+
+
+### Bug Fixes
+
+* Small fixes UI & Slack OAuth tuto ([#2398](https://github.com/windmill-labs/windmill/issues/2398)) ([e1eccc2](https://github.com/windmill-labs/windmill/commit/e1eccc2d9331ba4e33019a6109bc0368d718397c))
+
+## [1.182.0](https://github.com/windmill-labs/windmill/compare/v1.181.0...v1.182.0) (2023-10-10)
+
+
+### Features
+
+* add support for aggrid ee ([c4a817a](https://github.com/windmill-labs/windmill/commit/c4a817aeb6590d8972342f815f3cf3b891ea1446))
+* **frontend:** App polish ([#2397](https://github.com/windmill-labs/windmill/issues/2397)) ([11e0bc7](https://github.com/windmill-labs/windmill/commit/11e0bc76c4bc80339e43590f5becf6b2442a2227))
+* **frontend:** column definition helper ([#2399](https://github.com/windmill-labs/windmill/issues/2399)) ([53447f1](https://github.com/windmill-labs/windmill/commit/53447f1b43e897bb8856106cabc502c822052441))
+* **frontend:** error handler tutorial ([#2404](https://github.com/windmill-labs/windmill/issues/2404)) ([bc1ad3b](https://github.com/windmill-labs/windmill/commit/bc1ad3b8d09fb2b6547dbcb37ac074ffdf9b383c))
+* **frontend:** fix css editor + fix dark mode ([#2409](https://github.com/windmill-labs/windmill/issues/2409)) ([2d7712c](https://github.com/windmill-labs/windmill/commit/2d7712c02115006fe84cb323b3b3af99ac14ffdb))
+* manage cache and init scripts from worker group UI ([#2396](https://github.com/windmill-labs/windmill/issues/2396)) ([2c9ae41](https://github.com/windmill-labs/windmill/commit/2c9ae41706edc6570559d7d83864fb05c846c0c1))
+
+
+### Bug Fixes
+
+* add lsp absolute imports for deno in all cases ([27c45e3](https://github.com/windmill-labs/windmill/commit/27c45e38cc57350df193440aa0c09ddbca93902a))
+* fix aggrid initialization ([9b75e33](https://github.com/windmill-labs/windmill/commit/9b75e33887c3a9c4cac84d648763a6e3b4490fae))
+* **frontend:** Fix tutorial trigger ([#2392](https://github.com/windmill-labs/windmill/issues/2392)) ([cad37bc](https://github.com/windmill-labs/windmill/commit/cad37bc6defa7a42b96fec6ad0a9bcac55d88d51))
+* improve flow status viewer for large values ([64c5590](https://github.com/windmill-labs/windmill/commit/64c5590aa32e4dbff6af43e711cb6899c02e4ee3))
+* improve handling of large results by frontend ([21454a7](https://github.com/windmill-labs/windmill/commit/21454a7a052db3cc1d24fd36c4504098751c66d2))
+* tarball for workspace export is generated in /tmp/windmill ([f4957d6](https://github.com/windmill-labs/windmill/commit/f4957d66b9bf6124ad3f73912f32cd1ea47b46e2))
+
+## [1.181.0](https://github.com/windmill-labs/windmill/compare/v1.180.0...v1.181.0) (2023-10-05)
+
+
+### Features
+
+* add npm_config_registry support for bun, deno and being settable from UI ([#2373](https://github.com/windmill-labs/windmill/issues/2373)) ([c42b875](https://github.com/windmill-labs/windmill/commit/c42b8750f1d41c9b4de6c96f1ea82239c5325495))
+* **frontend:** add driverjs ([#2327](https://github.com/windmill-labs/windmill/issues/2327)) ([bda6f1f](https://github.com/windmill-labs/windmill/commit/bda6f1fe5d44a3c1d925c1b8a8e872d9f5fba484))
+
+
+### Bug Fixes
+
+* add numeric, array and date types ([#2379](https://github.com/windmill-labs/windmill/issues/2379)) ([768f972](https://github.com/windmill-labs/windmill/commit/768f972cbf578b3394f89120d172b02bcaac5413))
+* add reserved variables in args ([#2371](https://github.com/windmill-labs/windmill/issues/2371)) ([e7165f3](https://github.com/windmill-labs/windmill/commit/e7165f3357a2ba7a690accd78a03c2518aa61860))
+* ai flow prompt fix + explanation in ui ([#2374](https://github.com/windmill-labs/windmill/issues/2374)) ([66d15f0](https://github.com/windmill-labs/windmill/commit/66d15f0c17698077c5bf299af8368e9cfdbf3ecb))
+* flow trigger prompt + lower temp ([#2377](https://github.com/windmill-labs/windmill/issues/2377)) ([733bfe3](https://github.com/windmill-labs/windmill/commit/733bfe3f14e6eb0237c6a528ab64ae71082a4679))
+* **frontend:** fix flow tutorials ([#2383](https://github.com/windmill-labs/windmill/issues/2383)) ([63ad53f](https://github.com/windmill-labs/windmill/commit/63ad53fa70c4f1769d873bff962bfb2d66081163))
+* schema autocomplete/ai ([#2372](https://github.com/windmill-labs/windmill/issues/2372)) ([9ed748a](https://github.com/windmill-labs/windmill/commit/9ed748a0dac95f152f91de6e25b63d841af0dd50))
+* trigger bun prompt ([#2368](https://github.com/windmill-labs/windmill/issues/2368)) ([fc9adbe](https://github.com/windmill-labs/windmill/commit/fc9adbe56081065fa3de662e664fcebe0f4c25ee))
+
+## [1.180.0](https://github.com/windmill-labs/windmill/compare/v1.179.1...v1.180.0) (2023-10-01)
+
+
+### Features
+
+* code content search ([#2367](https://github.com/windmill-labs/windmill/issues/2367)) ([fb96059](https://github.com/windmill-labs/windmill/commit/fb960594fce265d5d4f4eb443e0c9cc19d14e025))
+
+
+### Bug Fixes
+
+* improve connection in apps ([a2fca17](https://github.com/windmill-labs/windmill/commit/a2fca17ae2ac8257154e2aec4a0ceabfe16fc46a))
+
+## [1.179.1](https://github.com/windmill-labs/windmill/compare/v1.179.0...v1.179.1) (2023-09-30)
+
+
+### Bug Fixes
+
+* fix 0 len flow module processing ([f97289a](https://github.com/windmill-labs/windmill/commit/f97289a3d8bc6ce978d0be1fec35a424211e4a20))
+
+## [1.179.0](https://github.com/windmill-labs/windmill/compare/v1.178.1...v1.179.0) (2023-09-30)
+
+
+### Features
+
+* add trustedDependencies escape hatch for bun ([#2364](https://github.com/windmill-labs/windmill/issues/2364)) ([52df265](https://github.com/windmill-labs/windmill/commit/52df2650ea5d5c03e94c96af0b8a79275856fc37))
+* ai code completion ([#2361](https://github.com/windmill-labs/windmill/issues/2361)) ([0937706](https://github.com/windmill-labs/windmill/commit/093770692ac40b8ee0139f24d63bcccda9bf6ddb))
+* **backend:** parse expires_in from string in TokenResponse ([#2353](https://github.com/windmill-labs/windmill/issues/2353)) ([4621915](https://github.com/windmill-labs/windmill/commit/46219154de07ef5a6e071f1c2859cea35c7f9943))
+* **frontend:** copy schema from json and past runs in flow inputs ([#2352](https://github.com/windmill-labs/windmill/issues/2352)) ([3cb2977](https://github.com/windmill-labs/windmill/commit/3cb29778dd70199d9504aa7c1a12bfd7a02569d6))
+
+
+### Bug Fixes
+
+* error handler does not recover flow anymore and error handler is called only once up the flow ([445bf96](https://github.com/windmill-labs/windmill/commit/445bf965eddc6da39a125fce60b53e0903698664))
+* **frontend:** Properly handle click ([#2351](https://github.com/windmill-labs/windmill/issues/2351)) ([55b7f98](https://github.com/windmill-labs/windmill/commit/55b7f982c2bbbb5d4daa9752ec8ffc0c79c374fc))
+* **frontend:** timezone fix ([#2360](https://github.com/windmill-labs/windmill/issues/2360)) ([dcfa5fc](https://github.com/windmill-labs/windmill/commit/dcfa5fc0e40f5cd8dba5a26be31695ce765c7e23))
+* improve superadmin settings page ([b029027](https://github.com/windmill-labs/windmill/commit/b029027c1c75c0b6489966371db7d2f9c99d15f8))
+* non skipped failures stop even in presence of an error handler ([1c5cc0c](https://github.com/windmill-labs/windmill/commit/1c5cc0c237101caf6c5e6e34b11c967a27cd4112))
+* remove shared http clients in rest runtime ([4931ed9](https://github.com/windmill-labs/windmill/commit/4931ed95c4b12f63effa1dd7d6a5cd526a612302))
+
+## [1.178.1](https://github.com/windmill-labs/windmill/compare/v1.178.0...v1.178.1) (2023-09-28)
+
+
+### Bug Fixes
+
+* improve license key check ([035bad5](https://github.com/windmill-labs/windmill/commit/035bad5268d182af3f30915b5356defd7f6ccbc0))
+
+## [1.178.0](https://github.com/windmill-labs/windmill/compare/v1.177.1...v1.178.0) (2023-09-28)
+
+
+### Features
+
+* **frontend:** add app groups management ([#2347](https://github.com/windmill-labs/windmill/issues/2347)) ([20e0427](https://github.com/windmill-labs/windmill/commit/20e0427a1303c1c32f41b198cd2d0f7f28b5bd32))
+* **frontend:** add AppDrawer controls ([#2339](https://github.com/windmill-labs/windmill/issues/2339)) ([3de6d44](https://github.com/windmill-labs/windmill/commit/3de6d446f281dcaac288deee19342a08e0ccf9af))
+* **frontend:** Switch to component list when deleting a component ([#2346](https://github.com/windmill-labs/windmill/issues/2346)) ([6fcd72c](https://github.com/windmill-labs/windmill/commit/6fcd72c79453dd9d60ca869cd9996cc0c25971fa))
+
+
+### Bug Fixes
+
+* add env tags to default worker group ([#2348](https://github.com/windmill-labs/windmill/issues/2348)) ([f5bed95](https://github.com/windmill-labs/windmill/commit/f5bed95ab15bc397f822b06816c43b4b13a84af3))
+
+## [1.177.1](https://github.com/windmill-labs/windmill/compare/v1.177.0...v1.177.1) (2023-09-26)
+
+
+### Bug Fixes
+
+* **frontend:** fix modal closing issues ([#2340](https://github.com/windmill-labs/windmill/issues/2340)) ([18cf8fa](https://github.com/windmill-labs/windmill/commit/18cf8faec16d496e4b327505b682459ed518a5b4))
+* **frontend:** fix overflow ([#2341](https://github.com/windmill-labs/windmill/issues/2341)) ([2e8f2ec](https://github.com/windmill-labs/windmill/commit/2e8f2ec724f6802170121f4f8aa73b697a39c9ee))
+* improve list component handling of non array data ([dc44b08](https://github.com/windmill-labs/windmill/commit/dc44b0841af17227160b9d56ec446e6646a8ab0d))
+
+## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
+
+
+### Features
+
+* add custom oauth support ([#2336](https://github.com/windmill-labs/windmill/issues/2336)) ([01277f4](https://github.com/windmill-labs/windmill/commit/01277f4d3b8bb04b955d5bbb2ed69c1c7c8f4f9e))
+* support automatic reconnection to pg ([ccaa05d](https://github.com/windmill-labs/windmill/commit/ccaa05d4bf5954c3fb8678239d2962cac6550a5a))
+
+
+### Bug Fixes
+
+* fix resource type picker object reinitialization ([f0f15c4](https://github.com/windmill-labs/windmill/commit/f0f15c47cb35cc1e3cfa13549465803a1e970770))
+* **frontend:** Fix build ([#2330](https://github.com/windmill-labs/windmill/issues/2330)) ([46592af](https://github.com/windmill-labs/windmill/commit/46592affd3d51b54632a2a7a281c11141edcb4a5))
+* **frontend:** Fix markdown dark mode ([#2329](https://github.com/windmill-labs/windmill/issues/2329)) ([6c19740](https://github.com/windmill-labs/windmill/commit/6c197407185810f43c47d4107007bd69814a1d65))
+* set min size of components to 1 ([d298093](https://github.com/windmill-labs/windmill/commit/d298093e29bd9983c7631a8f8c80e47b768bb93c))
+
+## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24)
+
+
+### Features
+
+* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254))
+* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6))
+* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33))
+* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85))
+* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8))
+
+
+### Bug Fixes
+
+* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0))
+* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64))
+* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1))
+* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9))
+* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781))
+* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55))
+* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9))
+* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91))
+
+## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19)
+
+
+### Features
+
+* add batch jobs ([#2306](https://github.com/windmill-labs/windmill/issues/2306)) ([5867e5d](https://github.com/windmill-labs/windmill/commit/5867e5d0f80fd515fab165659831b5ee9a8c3f97))
+* add dediacted worker env var ([#2296](https://github.com/windmill-labs/windmill/issues/2296)) ([e0c6eee](https://github.com/windmill-labs/windmill/commit/e0c6eee16e535b3a7d803a7978e463404f5fec30))
+* dedicated benchmarks ([#2297](https://github.com/windmill-labs/windmill/issues/2297)) ([c549239](https://github.com/windmill-labs/windmill/commit/c5492396843ddd9143ffe890696d0317c970de36))
+* **frontend:** Add component control doc ([#2295](https://github.com/windmill-labs/windmill/issues/2295)) ([26f8863](https://github.com/windmill-labs/windmill/commit/26f88636f0b972d4fe4931ed02135c38b27a56d2))
+* suggest adding openai key on workspace creation ([a6b3b2f](https://github.com/windmill-labs/windmill/commit/a6b3b2f63b317825a3d80218cbb606b9f610c221))
+* support pinned versions for bun in deployed scripts ([03806dc](https://github.com/windmill-labs/windmill/commit/03806dc3907cba724be14acb6aadf5be6e35cdb6))
+
+
+### Bug Fixes
+
+* add HOME to bun and deno ([0e3ecc7](https://github.com/windmill-labs/windmill/commit/0e3ecc7d6025c173135f20bacc33a0dc972ec222))
+* add queue_count to metrics ([9ced883](https://github.com/windmill-labs/windmill/commit/9ced8834a45151c6900b1eb33eca2cff4886a065))
+* ai improve prompts ([#2310](https://github.com/windmill-labs/windmill/issues/2310)) ([b647213](https://github.com/windmill-labs/windmill/commit/b647213b2c968b0cb1f90c97d94e8023c415dd55))
+* **frontend:** add missing key ([#2299](https://github.com/windmill-labs/windmill/issues/2299)) ([39d2467](https://github.com/windmill-labs/windmill/commit/39d24672ddd696372e55e9b4566f322a322385a8))
+* **frontend:** Always mount components ([#2309](https://github.com/windmill-labs/windmill/issues/2309)) ([34f94aa](https://github.com/windmill-labs/windmill/commit/34f94aa50e92254114c046fa8b7e900d93807937))
+* **frontend:** fix alignment ([#2307](https://github.com/windmill-labs/windmill/issues/2307)) ([f9fc6f1](https://github.com/windmill-labs/windmill/commit/f9fc6f19482e68c9ccba0014879fd8761662c36a))
+* **frontend:** Fix rich result styling + add title and hideDetails config ([#2294](https://github.com/windmill-labs/windmill/issues/2294)) ([732daef](https://github.com/windmill-labs/windmill/commit/732daef1c3515f7df3e09deac691bb585f9859cd))
+* **frontend:** fix tab styling + component bg ([#2308](https://github.com/windmill-labs/windmill/issues/2308)) ([5e773d3](https://github.com/windmill-labs/windmill/commit/5e773d386343f003425173207c166e3c4eeef956))
+* **frontend:** fix theme make default ([#2304](https://github.com/windmill-labs/windmill/issues/2304)) ([4629819](https://github.com/windmill-labs/windmill/commit/46298197c5333a81b9b8a004027ab9a856bdada4))
+* **frontend:** fix theme UI ([#2305](https://github.com/windmill-labs/windmill/issues/2305)) ([576f76b](https://github.com/windmill-labs/windmill/commit/576f76b1ffe9c50c8ccaca8c5e34d0ec03aebf3f))
+* validate more strongly usernames ([47094bb](https://github.com/windmill-labs/windmill/commit/47094bb8d1c6f4ba621d42515dede061fd04afdd))
+
+## [1.174.0](https://github.com/windmill-labs/windmill/compare/v1.173.0...v1.174.0) (2023-09-15)
+
+
+### Features
+
+* ai gen support all langs ([#2276](https://github.com/windmill-labs/windmill/issues/2276)) ([39590b3](https://github.com/windmill-labs/windmill/commit/39590b3d2592b2d08117c0f70829c13f1efb4885))
+* bun absolute/relative imports + tests ([#2286](https://github.com/windmill-labs/windmill/issues/2286)) ([e5ce85b](https://github.com/windmill-labs/windmill/commit/e5ce85b9affe665342f24b1d39ce3d03db09b941))
+* **frontend:** Global CSS editor ([#2178](https://github.com/windmill-labs/windmill/issues/2178)) ([7e9ee39](https://github.com/windmill-labs/windmill/commit/7e9ee39aa69bc31766b5e4f4aab498c8f14067cd))
+
+## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
+
+
+### Features
+
+* cli sync on windows ([#2283](https://github.com/windmill-labs/windmill/issues/2283)) ([c371cb3](https://github.com/windmill-labs/windmill/commit/c371cb397ab3d0c534e2c553d1dfb1ad5176d2a6))
+
+
+### Bug Fixes
+
+* accept jobs whose duration > 24 days ([2c00894](https://github.com/windmill-labs/windmill/commit/2c00894122aa8caee59b20625935284de6902950))
+
+## [1.172.1](https://github.com/windmill-labs/windmill/compare/v1.172.0...v1.172.1) (2023-09-14)
+
+
+### Bug Fixes
+
+* improve splitpane + improve deleting conditional tab ([1629008](https://github.com/windmill-labs/windmill/commit/1629008eb2eb48ff9cc2cf6b3a351efcf682244d))
+* update to svelte 4 ([#2280](https://github.com/windmill-labs/windmill/issues/2280)) ([90c10d8](https://github.com/windmill-labs/windmill/commit/90c10d803b4c47a9e1ac5b9e49e2a614344299a9))
+
+## [1.172.0](https://github.com/windmill-labs/windmill/compare/v1.171.0...v1.172.0) (2023-09-13)
+
+
+### Features
+
+* improve ai flow ([#2270](https://github.com/windmill-labs/windmill/issues/2270)) ([b23417a](https://github.com/windmill-labs/windmill/commit/b23417ab5b9938bbdf9db6449102760ff8c80152))
+* worker groups admin panel ([#2277](https://github.com/windmill-labs/windmill/issues/2277)) ([070b162](https://github.com/windmill-labs/windmill/commit/070b16222bc666866284180b3878f4d4f27bfa85))
+
+
+### Bug Fixes
+
+* ai flow nits ([#2272](https://github.com/windmill-labs/windmill/issues/2272)) ([8f6f46d](https://github.com/windmill-labs/windmill/commit/8f6f46de199d58133b9faa77cdbcbcfd6cb962f7))
+
+## [1.171.0](https://github.com/windmill-labs/windmill/compare/v1.170.0...v1.171.0) (2023-09-12)
+
+
+### Features
+
+* attempt to SIGTERM before SIGKILL for bash ([f40bbba](https://github.com/windmill-labs/windmill/commit/f40bbba519a97cbb1ec142c335f038dbebcd4e7c))
+* zero copy result for job result ([#2263](https://github.com/windmill-labs/windmill/issues/2263)) ([22a7da5](https://github.com/windmill-labs/windmill/commit/22a7da58b1d20721892906cba2dee6fbeb1cc1fd))
+
+
+### Bug Fixes
+
+* 2257 TIME  convertion in pg_executor.rs ([#2267](https://github.com/windmill-labs/windmill/issues/2267)) ([3d71253](https://github.com/windmill-labs/windmill/commit/3d71253abdb0dff1670a796d07a53ecd0a98414e))
+* fix field duplicate in app background settings ([164cdaf](https://github.com/windmill-labs/windmill/commit/164cdaf09464646dee4e70a699222a454eb0d898))
+* improve bun lockfile resolution ([9103ec4](https://github.com/windmill-labs/windmill/commit/9103ec445db81395a5851202eecb87301d0b4987))
+* remove result and args from list completed and list queue jobs ([e7e63e1](https://github.com/windmill-labs/windmill/commit/e7e63e111a73e0986050a8fe7fdc18784ba902b0))
+
+## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
+
+
+### Features
+
+* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
+* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
+
+
+### Bug Fixes
+
+* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
+
+## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
+
+
+### Features
+
+* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
+* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
+* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
+* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
+* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
+
+
+### Bug Fixes
+
+* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
+* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
+
+## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
+
+
+### Bug Fixes
+
+* add list resource types names ([fbbab5c](https://github.com/windmill-labs/windmill/commit/fbbab5c874748547a9ff3e58c1b7b22c90766f4f))
+* add stable ids to rows in AppTable ([0c91581](https://github.com/windmill-labs/windmill/commit/0c91581fcdf3a141f36e34610935aa100fcfee52))
+* reduce aggregate period to list users in workspace ([6bc0e37](https://github.com/windmill-labs/windmill/commit/6bc0e373fc6088636f09d217e8800a32337291ea))
+
 ## [1.168.2](https://github.com/windmill-labs/windmill/compare/v1.168.1...v1.168.2) (2023-09-06)
 
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:buster-slim as nsjail
+FROM debian:bookworm-slim as nsjail
 
 WORKDIR /nsjail
 
@@ -6,29 +6,29 @@ ARG nsjail=""
 
 RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
-    bison=2:3.3.* \
+    bison=2:3.8.* \
     flex=2.6.* \
-    g++=4:8.3.* \
-    gcc=4:8.3.* \
-    git=1:2.20.* \
-    libprotobuf-dev=3.6.* \
-    libnl-route-3-dev=3.4.* \
-    make=4.2.* \
-    pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*; fi
+    g++=4:12.2.* \
+    gcc=4:12.2.* \
+    git=1:2.39.* \
+    libprotobuf-dev=3.21.* \
+    libnl-route-3-dev=3.7.* \
+    make=4.3-4.1 \
+    pkg-config=1.8.* \
+    protobuf-compiler=3.21.*; fi
 
 
 RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
 RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
-FROM rust:slim-buster AS rust_base
+FROM rust:slim-bookworm AS rust_base
 
 RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl nodejs npm
+    curl nodejs
 
 RUN rustup component add rustfmt
 
@@ -87,7 +87,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:buster-slim as downloader
+FROM debian:bookworm-slim as downloader
 
 ARG TARGETPLATFORM
 
@@ -101,7 +101,7 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol
 
 RUN unzip deno.zip && rm deno.zip
 
-FROM python:3.11.4-slim-buster
+FROM python:3.11.4-slim-bookworm
 
 ARG TARGETPLATFORM
 
@@ -164,6 +164,9 @@ RUN set -eux; \
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
+# go build is slower the first time it is ran, so we prewarm it in the build
+RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm &&  printf "package foo\nimport (\"fmt\")\nfunc main() { fmt.Println(42) }" > warm.go && go build -x && rm -rf /tmp/gobuildwarm
+
 ENV TZ=Etc/UTC
 
 RUN /usr/local/bin/python3 -m pip install pip-tools
@@ -177,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.4 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

LICENSE

@@ -1,16 +1,22 @@
-
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
+./LICENSE-AGPL)
 
-Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
-Every file is under License AGPL unless otherwise specified 
-or belonging to one of the below cases:
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
+specified. Every file is under License AGPL unless otherwise specified or
+belonging to one of the below cases:
 
-The files under backend/ are AGPLv3 Licensed.
-The files under frontend/ are AGPLv3 Licensed.
-The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+The files under backend/ are AGPLv3 Licensed, except any snippets of code under
+the compile flag "enterprise". Those snippets and files are under a proprietary
+and commercial license. The files under frontend/ are AGPLv3 Licensed, except
+any snippets of code that require a positive license check to be activated.
+Those snippets and files are under a proprietary and commercial license. Private
+and public forks MUST not include any of the above proprietary and commercial
+code. Windmill Labs, Inc. provide tools to clean the codebase from those
+snippets upon demand. The files under python-client/ deno-client/ go-client/ are
+Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
 
-All third party components incorporated into the Windmill Software are licensed under the
-original license provided by the owner of the applicable component.
+All third party components incorporated into the Windmill Software are licensed
+under the original license provided by the owner of the applicable component.

README.md

@@ -64,6 +64,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
     - [OAuth for self-hosting](#oauth-for-self-hosting)
     - [smtp for self-hosting](#smtp-for-self-hosting)
     - [Resource types](#resource-types)
+    - [Manually fetch latest Windmill binary](#manually-fetch-latest-windmill-binary)
   - [Environment Variables](#environment-variables)
   - [Run a local dev setup](#run-a-local-dev-setup)
     - [only Frontend](#only-frontend)
@@ -211,7 +212,6 @@ compiling from source or using without a postgres super user, see
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/oauth.json -o oauth.json
 
 docker compose up -d
 ```
@@ -227,11 +227,21 @@ From there, you can follow the setup app and create other users.
 We publish helm charts at:
 <https://github.com/windmill-labs/windmill-helm-charts>.
 
-### Postgres without superuser
+### Run from binaries
 
-If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
-postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for Windmill.
+Each release includes the corresponding binaries for x86_64. You can simply download the 
+latest `windmill` binary using the following set of bash commands.
+```bash
+BINARY_NAME='windmill-amd64' # or windmill-ee-amd64 for the enterprise edition
+LATEST_RELEASE=$(curl -L -s -H 'Accept: application/json' https://github.com/windmill-labs/windmill/releases/latest)
+LATEST_VERSION=$(echo $LATEST_RELEASE | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+ARTIFACT_URL="https://github.com/windmill-labs/windmill/releases/download/$LATEST_VERSION/$BINARY_NAME"
+wget "$ARTIFACT_URL" -O windmill
+```
+
+### OAuth, SSO & SMTP
+
+Windmill Community Edition allows to configure the OAuth, SSO (including Google Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin settings. Do note that there is a limit of 50 SSO users on the community edition.
 
 ### Commercial license
 
@@ -250,91 +260,21 @@ your current infrastructure to Windmill, support with tight SLA, and our global
 cache sync for high-performance/no dependency cache miss of cluster from 10+
 nodes to 200+ nodes.
 
-### OAuth for self-hosting
-
-To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
-the following format:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    "allowed_domains": ["windmill.dev"] //restrict a client OAuth login to some domains
-  }
-}
-```
-
-and mount it at `/usr/src/app/oauth.json`.
-
-The redirect url for the oauth clients is:
-`<instance_url>/user/login_callback/<client>`
-
-Even if you setup oauth, you will still want to **login as admin@windmill.dev /
-changeme** to setup your instance as a super-admin and give yourself admin
-rights.
-
-[The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
-
-To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://www.windmill.dev/docs/misc/contributing). We
-welcome contributions!
-
-You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    // To add a new OAuth2 IdP
-    "login_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "userinfo_url": "<userinfo endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    },
-    // To add a new OAuth2 Resource
-    "connect_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    }
-  }
-}
-```
-
-### smtp for self-hosting
-
-For users to receive emails when you invite them to workspaces or add them to
-the instances using their emails, configure the SMTP env variables in the
-servers:
-
-```
-SMTP_FROM=noreply@windmill.dev
-SMTP_HOST=smtp.gmail.com
-SMTP_PORT=587
-SMTP_USERNAME=ruben@windmill.dev
-SMTP_PASSWORD=yourpasswordapp
-```
-
 ### Resource types
 
 You will also want to import all the approved resource types from
 [WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
 it being synced automatically everyday.
 
+
 ## Environment Variables
 
 | Environment Variable name                     | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
 | --------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL                                  |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| WORKER_GROUP                                  | default                                    | The worker group the worker belongs to and get its configuration pulled from                                                                                                                       | Worker                |
 | SERVER_BIND_ADDR                              | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
 | PORT                                          | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |
 | METRICS_ADDR                                  | None                                       | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                      | All                   |
 | JSON_FMT                                      | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
@@ -374,9 +314,11 @@ it being synced automatically everyday.
 | PIP_LOCAL_DEPENDENCIES                        | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
 | ADDITIONAL_PYTHON_PATHS                       | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
 | INCLUDE_HEADERS                               | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path>                                                                                                            | Worker                |
 | CUSTOM_TAGS                                   | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
 | JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
 | WAIT_RESULT_FAST_POLL_INTERVAL_MS             | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
@@ -395,12 +337,13 @@ it being synced automatically everyday.
 | SMTP_USERNAME                                 | None                                       | username for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
-| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | false                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
+| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                       | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
 | GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |
 | SECRET_SALT                                   | None                                       | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server    | Server + Worker       |
 | OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI API base path (no trailing slash)                                                                                                                                                     | Server                |
+| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
 
 ## Run a local dev setup
 

backend/.sqlx/query-025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM global_settings WHERE name = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0"
+}

backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = $1 WHERE workspace_id = $2",
+  "query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Varchar",
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "d7d368f9ef2587c6abd3daaec93401615a53e3d7d949b65ebf01ef8760ba46b8"
+  "hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304"
 }

backend/.sqlx/query-03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
+}

backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM config WHERE name = $1 RETURNING name",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a"
+}

backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT script_path FROM completed_job  WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "script_path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd"
+}

backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597"

backend/.sqlx/query-240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -37,6 +37,11 @@
         "ordinal": 6,
         "name": "custom_tags",
         "type_info": "TextArray"
+      },
+      {
+        "ordinal": 7,
+        "name": "worker_group",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -52,8 +57,9 @@
       false,
       false,
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "4f6b3b472b4b78c0325cf3755f9ef1806d2e82328ceccbeade8cc2333c6dfe47"
+  "hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
 }

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration_ms",
-        "type_info": "Int4"
+        "type_info": "Int8"
       }
     ],
     "parameters": {

backend/.sqlx/query-3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, count(*) as count FROM queue WHERE\n        scheduled_for <= now() - ('3 seconds')::interval AND running = false\n        GROUP BY tag",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a"
+}

backend/.sqlx/query-47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Int4",
+        "TextArray",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32"
+  "hash": "47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f"
 }

backend/.sqlx/query-502781c4e2fc692db7a66b850450a1934b2403a5cff0421fe9609f9b99d8ee95.json

@@ -11,7 +11,7 @@
     ],
     "parameters": {
       "Left": [
-        "Int4"
+        "Int8"
       ]
     },
     "nullable": [

backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"

backend/.sqlx/query-584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a"
+}

backend/.sqlx/query-5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT $1 OFFSET $2",
+  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -49,5 +49,5 @@
       true
     ]
   },
-  "hash": "77ba7207c8f5fd7156542cfd9943aa9a9fa87a652131c261f5020bab9ba6b5a3"
+  "hash": "5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c"
 }

backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "TextArray"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf"
-}

backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json

@@ -1,14 +1,15 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = NULL WHERE workspace_id = $1",
+  "query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "28eef409e9f25e4c037dd716d6515e96e8962acb59d5e64f9cc3227e0f5fd058"
+  "hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03"
 }

backend/.sqlx/query-644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e"
+}

backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json

@@ -0,0 +1,74 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH uuid_table as (\n            select gen_random_uuid() as uuid from generate_series(1, $11)\n        )\n        INSERT INTO queue \n            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n            (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n        RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d"
+}

backend/.sqlx/query-6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86"
+}

backend/.sqlx/query-8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT COUNT(*) FROM group_ WHERE name != 'all'",
+  "query": "SELECT COUNT(*) FROM password WHERE login_type != 'password'",
   "describe": {
     "columns": [
       {
@@ -16,5 +16,5 @@
       null
     ]
   },
-  "hash": "18bdd06f4791c652addb4783f419f5d020c1d23278f583c6be6314768f9bf25a"
+  "hash": "8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483"
 }

backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = 'server'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+}

backend/.sqlx/query-8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM resource_type WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6"
+}

backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "avg_duration_s",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1"
-}

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -0,0 +1,66 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e"
+}

backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47"
+}

backend/.sqlx/query-a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "avg_duration_s",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0"
+}

backend/.sqlx/query-a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108"
+}

backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT progress::bigint FROM tutorial_progress WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "progress",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2"
+}

backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49"
+}

backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429"
+}

backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name LIKE 'worker__%'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7"
+}

backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "openai_resource_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      false
+    ]
+  },
+  "hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee"
+}

backend/.sqlx/query-cec906e3bcfe5b477f23e8bb44500b97166c789ab1cf6487679d092eebf3a48b.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM group_ WHERE name != 'all' AND name != 'error_handler' AND name != 'slack'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "cec906e3bcfe5b477f23e8bb44500b97166c789ab1cf6487679d092eebf3a48b"
+}

backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9"
+}

backend/.sqlx/query-d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4"
+}

backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07"
+}

backend/.sqlx/query-e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468"
+}

backend/.sqlx/query-ed5764ead5e8f433adfe666976664876b22dcb2b34dcd99f1e44cd03cb9ab4ef.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      false
+    ]
+  },
+  "hash": "ed5764ead5e8f433adfe666976664876b22dcb2b34dcd99f1e44cd03cb9ab4ef"
+}

backend/.sqlx/query-ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
+}

backend/.sqlx/query-efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb"
+}

backend/.sqlx/query-f516ca558816c2cbab3c8ae865ef8f764aa4686e7df9c53d30d49a3dbbf36af4.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration",
-        "type_info": "Int8"
+        "type_info": "Numeric"
       }
     ],
     "parameters": {

backend/.sqlx/query-f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "database_length!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
+}

backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.168.2"
+version = "1.183.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.168.2"
+version = "1.183.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -54,7 +54,6 @@ tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
 git-version.workspace = true
-rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
@@ -64,6 +63,8 @@ once_cell.workspace = true
 prometheus.workspace = true
 uuid.workspace = true
 gethostname.workspace = true
+serde_json.workspace = true
+serde.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -96,7 +97,7 @@ tower = "^0"
 tower-http = { version = "^0", features = ["trace", "cors"] }
 tower-cookies = "^0"
 serde = "^1"
-serde_json = { version = "^1", features = ["preserve_order"] }
+serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
@@ -112,7 +113,7 @@ mime_guess = "^2"
 hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
-retainer = "^0"
+quick_cache = "^0"
 rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
@@ -155,7 +156,9 @@ sqlx = { version = "^0", features = [
     "chrono",
     "postgres",
     "runtime-tokio-rustls",
+    "bigdecimal"
 ] }
+bigdecimal = "^0"
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
@@ -187,6 +190,7 @@ postgres-native-tls = "^0"
 native-tls = "^0"
 samael = { version = "0.0.12", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
-rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
+rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
 pem = "3.0.1"
+nix = { version = "0.27.1", features = ["process", "signal"] }

backend/migrations/20230908155455_add_worker_group.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE worker_group_config;
+
+ALTER TABLE worker_ping DROP COLUMN worker_group;

backend/migrations/20230908155455_add_worker_group.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE worker_group_config (
+    name VARCHAR(255) PRIMARY KEY,
+    config JSONB DEFAULT '{}'::jsonb
+);
+
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS worker_group VARCHAR(255) NOT NULL DEFAULT 'default';
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS dedicated_worker VARCHAR(255);

backend/migrations/20230914062021_fix_group_grant_roles.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914062021_fix_group_grant_roles.up.sql

@@ -0,0 +1 @@
+-- Add up migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE completed_job ALTER COLUMN duration_ms TYPE bigint;
+ALTER TABLE completed_job ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN canceled_by TYPE VARCHAR(255);

backend/migrations/20230915084031_add_themes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230915084031_add_themes.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;
+INSERT INTO resource (workspace_id, path, value, description, resource_type) SELECT id, 'f/app_themes/theme_0', '{"name": "Default Theme", "value": ""}', 'The default app theme', 'app_theme' FROM workspace ON CONFLICT DO NOTHING

backend/migrations/20230917075933_rename_config.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE IF exists config RENAME TO worker_group_config ;

backend/migrations/20230917075933_rename_config.up.sql

@@ -0,0 +1,29 @@
+-- Add up migration script here
+ALTER TABLE IF exists worker_group_config RENAME TO config;
+UPDATE config SET name = 'worker__' || name;
+
+CREATE FUNCTION "notify_config_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_config_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_config_change"
+ AFTER INSERT OR UPDATE ON "config"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_config_change" ();
+
+CREATE FUNCTION "notify_global_setting_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_global_setting_change"
+ AFTER INSERT OR UPDATE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_change" ();

backend/migrations/20230921084441_add_default_worker_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230921084441_add_default_worker_groups.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+INSERT INTO config (name, config) VALUES  
+    ('worker__default', '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun"]}'), 
+    ('worker__native', '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql", "snowflake"]}') ON CONFLICT DO NOTHING;

backend/migrations/20230927215813_add_app_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230927215813_add_app_groups.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20230929022202_tutorials.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE tutorial_progress

backend/migrations/20230929022202_tutorials.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE  tutorial_progress(
+    email VARCHAR(255) PRIMARY KEY,
+    progress bit(64) NOT NULL DEFAULT B'0'
+);
+
+GRANT ALL ON tutorial_progress TO windmill_admin;
+GRANT ALL ON tutorial_progress TO windmill_user;

backend/migrations/20230929092213_copilot_code_completion.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN code_completion_enabled;

backend/migrations/20230929092213_copilot_code_completion.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN code_completion_enabled BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20231002123723_notify_delete_config.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231002123723_notify_delete_config.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+
+CREATE FUNCTION "notify_global_setting_delete" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', OLD.name::text);
+    RETURN OLD;
+END;
+$$ LANGUAGE PLPGSQL;
+  
+CREATE OR REPLACE TRIGGER "notify_global_setting_delete"
+ AFTER DELETE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_delete" ();
+

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -35,6 +35,10 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "dateutil" => "python-dateutil",
     "mailparser" => "mail-parser",
     "mailparser-reply" => "mail-parser-reply",
+    "gitlab" => "python-gitlab",
+    "smbclient" => "smbprotocol",
+    "playhouse" => "peewee",
+    "dns" => "dnspython"
 };
 
 fn replace_import(x: String) -> String {

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -48,7 +48,7 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
 }
 
 lazy_static::lazy_static! {
-    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+))?"#).unwrap();
+    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+(?:\[\])?))?"#).unwrap();
 
     // -- $1 name (type) = default
     static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
@@ -213,27 +213,35 @@ pub fn parse_mysql_typ(typ: &str) -> Typ {
 }
 
 pub fn parse_pg_typ(typ: &str) -> Typ {
-    match typ {
-        "varchar" => Typ::Str(None),
-        "text" => Typ::Str(None),
-        "int" => Typ::Int,
-        "bigint" => Typ::Int,
-        "bool" => Typ::Bool,
-        "char" => Typ::Str(None),
-        "smallint" => Typ::Int,
-        "smallserial" => Typ::Int,
-        "serial" => Typ::Int,
-        "bigserial" => Typ::Int,
-        "real" => Typ::Float,
-        "double precision" => Typ::Float,
-        "oid" => Typ::Int,
-        _ => Typ::Str(None),
+    if typ.ends_with("[]") {
+        let base_typ = parse_pg_typ(typ.strip_suffix("[]").unwrap());
+        Typ::List(Box::new(base_typ))
+    } else {
+        match typ {
+            "varchar" => Typ::Str(None),
+            "text" => Typ::Str(None),
+            "int" => Typ::Int,
+            "bigint" => Typ::Int,
+            "bool" => Typ::Bool,
+            "char" => Typ::Str(None),
+            "smallint" => Typ::Int,
+            "smallserial" => Typ::Int,
+            "serial" => Typ::Int,
+            "bigserial" => Typ::Int,
+            "real" => Typ::Float,
+            "double precision" => Typ::Float,
+            "numeric" => Typ::Float,
+            "decimal" => Typ::Float,
+            "oid" => Typ::Int,
+            "date" | "time" | "timestamp" => Typ::Datetime,
+            _ => Typ::Str(None),
+        }
     }
 }
 
 pub fn parse_bigquery_typ(typ: &str) -> Typ {
     if typ.ends_with("[]") {
-        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap_or(typ));
+        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap());
         Typ::List(Box::new(base_typ))
     } else {
         match typ {

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.154.3",
+  "version": "1.180.0",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -97,6 +97,15 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
 function addHeapObject(obj) {
     if (heap_next === heap.length) heap.push(heap.length + 1);
     const idx = heap_next;
@@ -115,15 +124,6 @@ function getFloat64Memory0() {
     return cachedFloat64Memory0;
 }
 
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
-}
-
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -492,13 +492,13 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
-    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
+    imports.wbg.__wbg_eval_2e5c54091872c19c = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -507,6 +507,10 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -543,10 +547,6 @@ function __wbg_get_imports() {
         const ret = BigInt.asUintN(64, arg0);
         return addHeapObject(ret);
     };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;

backend/src/ee.rs

@@ -1,41 +1,49 @@
 #[cfg(feature = "enterprise")]
-use base64::Engine;
-#[cfg(feature = "enterprise")]
-use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
-#[cfg(feature = "enterprise")]
-use sha2::Sha256;
+use windmill_common::error;
 
-#[cfg(feature = "enterprise")]
-pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
-    if let Some(license_key) = license_key {
-        let mut splitted_lk = license_key.split(".");
-        if splitted_lk.clone().count() != 3 {
-            panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
-        }
-        let id = splitted_lk.next().unwrap();
-        let expiry = splitted_lk.next().unwrap();
-        let signature_b64 = splitted_lk.next().unwrap();
+pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
+    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
 
-        let expiry_nb = expiry.parse::<u64>()?;
-        if expiry_nb < chrono::Utc::now().timestamp() as u64 {
-            panic!(
-                "License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
-                chrono::Utc::now().timestamp()
-            );
-        }
-        const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
-        let pub_key = rsa::RsaPublicKey::from_public_key_der(
-            &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
-        )?;
-        let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
-        rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
-            .verify(
-                &format!("{id}{expiry}").as_bytes(),
-                &rsa::pss::Signature::from(signature),
-            )
-            .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
-    } else {
-        panic!("License key is required for the enterprise edition");
+    let id = validate_license_key(license_key.clone()).await?;
+    {
+        let mut l = LICENSE_KEY_ID.write().await;
+        *l = id.to_string()
     }
+
+    {
+        let mut l = LICENSE_KEY.write().await;
+        *l = license_key
+    }
+    {
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = true
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn verify_license_key() -> error::Result<()> {
+    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::error::to_anyhow;
+
+    let expiry_nb = LICENSE_KEY
+        .read()
+        .await
+        .clone()
+        .split(".")
+        .nth(1)
+        .unwrap_or_else(|| "")
+        .parse::<u64>()
+        .map_err(to_anyhow)?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = false;
+    };
     Ok(())
 }

backend/src/main.rs

@@ -8,19 +8,27 @@
 
 use gethostname::gethostname;
 use git_version::git_version;
-use monitor::handle_zombie_jobs_periodically;
-use sqlx::{Pool, Postgres};
+use rand::Rng;
+use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     sync::Arc,
+    time::Duration,
 };
 use tokio::{
     fs::{metadata, DirBuilder},
-    join,
     sync::RwLock,
 };
-use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
+use windmill_common::{
+    global_settings::{
+        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, EXTRA_PIP_INDEX_URL_SETTING,
+        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
+    },
+    utils::rd_string,
+    worker::{reload_custom_tags_setting, WORKER_GROUP},
+    DB, METRICS_ADDR,
+};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -28,6 +36,12 @@ use windmill_worker::{
     PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
 
+use crate::monitor::{
+    initial_load, monitor_db, reload_base_url_setting, reload_extra_pip_index_url_setting,
+    reload_license_key, reload_npm_config_registry_setting, reload_retention_period_setting,
+    reload_server_config, reload_worker_config,
+};
+
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_PORT: u16 = 8000;
@@ -126,16 +140,9 @@ Windmill Community Edition {GIT_VERSION}
 
     display_config(&ENV_SETTINGS);
 
-    tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
-    if let Some(ref smtp) = *SMTP_CLIENT {
-        tracing::info!("Smtp client defined. Testing connection...");
-        if let Err(e) = smtp.connect().await {
-            tracing::error!("Failed to connect to smtp server: {}", e);
-        } else {
-            tracing::info!("Smtp client connected.");
-        }
-    }
-    if server_mode || num_workers > 0 {
+    let worker_mode = num_workers > 0;
+
+    if server_mode || worker_mode {
         let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
 
         let port = if server_mode {
@@ -144,6 +151,13 @@ Windmill Community Edition {GIT_VERSION}
             port_var.unwrap_or(0)
         };
 
+        // since it's only on server mode, the port is statically defined
+        let base_internal_url: String = format!("http://localhost:{}", port.to_string());
+
+        initial_load(&db, tx.clone(), worker_mode, server_mode).await;
+
+        monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
+
         if std::env::var("BASE_INTERNAL_URL").is_ok() {
             tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
         }
@@ -153,6 +167,12 @@ Windmill Community Edition {GIT_VERSION}
         let rsmq2 = rsmq.clone();
         let (port_tx, port_rx) = tokio::sync::oneshot::channel::<u16>();
 
+        DirBuilder::new()
+            .recursive(true)
+            .create("/tmp/windmill")
+            .await
+            .expect("could not create initial server dir");
+
         let server_f = async {
             windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe(), port_tx).await?;
             Ok(()) as anyhow::Result<()>
@@ -161,10 +181,11 @@ Windmill Community Edition {GIT_VERSION}
         let workers_f = async {
             let port = port_rx.await?;
             let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-            if num_workers > 0 {
+            if worker_mode {
                 run_workers(
                     db.clone(),
                     rx.resubscribe(),
+                    tx.clone(),
                     num_workers,
                     base_internal_url.clone(),
                     rsmq.clone(),
@@ -176,29 +197,130 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
-        let rsmq2 = rsmq.clone();
         let monitor_f = async {
-            if server_mode {
-                // since it's only on server mode, the port is statically defined
-                let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-                monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2).await;
+            let db = db.clone();
+            let tx = tx.clone();
+            let rsmq = rsmq.clone();
+
+            let mut rx = rx.resubscribe();
+            let base_internal_url = base_internal_url.to_string();
+            let h = tokio::spawn(async move {
+                let mut listener = retry_listen_pg(&db).await;
+
+                loop {
+                    tokio::select! {
+                        _ = tokio::time::sleep(Duration::from_secs(30))    => {
+                            monitor_db(
+                                &db,
+                                &base_internal_url,
+                                rsmq.clone(),
+                                server_mode,
+                            )
+                            .await;
+                        },
+                        notification = listener.recv() => {
+                            match notification {
+                                Ok(n) => {
+                                    tracing::info!("Received new pg notification: {n:?}");
+                                    match n.channel() {
+                                        "notify_config_change" => {
+                                            match n.payload() {
+                                                "server" if server_mode => {
+                                                    tracing::info!("Server config change detected: {}", n.payload());
+
+                                                    reload_server_config(&db).await;
+                                                },
+                                                a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
+                                                    tracing::info!("Worker config change detected: {}", n.payload());
+                                                    reload_worker_config(&db, tx.clone(), true).await;
+                                                },
+                                                _ => {
+                                                    tracing::debug!("config changed but did not target this server/worker");
+                                                }
+                                            }
+                                        },
+                                        "notify_global_setting_change" => {
+                                            tracing::info!("Global setting change detected: {}", n.payload());
+                                            match n.payload() {
+                                                BASE_URL_SETTING => {
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload base url setting");
+                                                    }
+                                                },
+                                                OAUTH_SETTING => {
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload oauth setting");
+                                                    }
+                                                },
+                                                CUSTOM_TAGS_SETTING => {
+                                                    if let Err(e) = reload_custom_tags_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload custom tags setting");
+                                                    }
+                                                },
+                                                LICENSE_KEY_SETTING => {
+                                                    if let Err(e) = reload_license_key(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload license key setting");
+                                                    }
+                                                },
+                                                RETENTION_PERIOD_SECS_SETTING => {
+                                                    reload_retention_period_setting(&db).await
+                                                },
+                                                EXTRA_PIP_INDEX_URL_SETTING => {
+                                                    reload_extra_pip_index_url_setting(&db).await
+                                                },
+                                                NPM_CONFIG_REGISTRY_SETTING => {
+                                                    reload_npm_config_registry_setting(&db).await
+                                                },
+                                                REQUEST_SIZE_LIMIT_SETTING => {
+                                                    tracing::info!("Request limit size change detected, killing server expecting to be restarted");
+                                                    // we wait a bit randomly to avoid having all servers shutdown at same time
+                                                    let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                    if let Err(e) = tx.send(()) {
+                                                        tracing::error!(error = %e, "Could not send killpill to server");
+                                                    }
+                                                }
+                                                a @_ => {
+                                                    tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
+                                                }
+                                            }
+                                        },
+                                        _ => {
+                                            tracing::warn!("Unknown notification received");
+                                            continue;
+                                        }
+                                    }
+                                },
+                                Err(e) => {
+                                    tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
+                                    listener = retry_listen_pg(&db).await;
+                                    continue;
+                                }
+                            };
+                        },
+                        _ = rx.recv() => {
+                                println!("received killpill for monitor job");
+                                break;
+                        }
+                    }
+                }
+            });
+
+            if let Err(e) = h.await {
+                tracing::error!("Error waiting for monitor handle:{e}")
             }
             Ok(()) as anyhow::Result<()>
         };
 
         let metrics_f = async {
-            match metrics_addr {
-                Some(_addr) => {
-                    #[cfg(not(feature = "enterprise"))]
-                    panic!("Metrics are only available in the Enterprise Edition");
+            if let Some(_addr) = metrics_addr {
+                #[cfg(not(feature = "enterprise"))]
+                panic!("Metrics are only available in the Enterprise Edition");
 
-                    #[cfg(feature = "enterprise")]
-                    windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
-                        .await
-                        .map_err(anyhow::Error::from)
-                }
-                None => Ok(()),
+                #[cfg(feature = "enterprise")]
+                windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
             }
+            Ok(()) as anyhow::Result<()>
         };
 
         futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
@@ -208,6 +330,40 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
+async fn listen_pg(db: &DB) -> Option<PgListener> {
+    let mut listener = match PgListener::connect_with(&db).await {
+        Ok(l) => l,
+        Err(e) => {
+            tracing::error!(error = %e, "Could not connect to database");
+            return None;
+        }
+    };
+
+    if let Err(e) = listener
+        .listen_all(vec!["notify_config_change", "notify_global_setting_change"])
+        .await
+    {
+        tracing::error!(error = %e, "Could not listen to database");
+        return None;
+    }
+
+    return Some(listener);
+}
+
+async fn retry_listen_pg(db: &DB) -> PgListener {
+    let mut listener = listen_pg(db).await;
+    loop {
+        if listener.is_none() {
+            tracing::info!("Retrying listening to pg listen in 5 seconds");
+            tokio::time::sleep(Duration::from_secs(5)).await;
+            listener = listen_pg(db).await;
+        } else {
+            tracing::info!("Successfully connected to pg listen");
+            return listener.unwrap();
+        }
+    }
+}
+
 fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
@@ -225,40 +381,14 @@ fn display_config(envs: &[&str]) {
     )
 }
 
-pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
-    db: &Pool<Postgres>,
-    rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
-) -> tokio::task::JoinHandle<()> {
-    let db1 = db.clone();
-    let db2 = db.clone();
-
-    let rx2 = rx.resubscribe();
-    let base_internal_url = base_internal_url.to_string();
-    tokio::spawn(async move {
-        join!(
-            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
-            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
-        );
-    })
-}
-
 pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
+    tx: tokio::sync::broadcast::Sender<()>,
     num_workers: i32,
     base_internal_url: String,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
-    #[cfg(feature = "enterprise")]
-    ee::verify_license_key(LICENSE_KEY.clone())?;
-
-    #[cfg(not(feature = "enterprise"))]
-    if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) {
-        panic!("License key is required ONLY for the enterprise edition");
-    }
-
     let instance_name = gethostname()
         .to_str()
         .map(|x| {
@@ -320,6 +450,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
         let ip = ip.clone();
         let rx = rx.resubscribe();
+        let tx = tx.clone();
         let base_internal_url = base_internal_url.clone();
         let rsmq2 = rsmq.clone();
         let sync_barrier = sync_barrier.clone();
@@ -333,6 +464,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 num_workers as u32,
                 &ip,
                 rx,
+                tx,
                 &base_internal_url,
                 rsmq2,
                 sync_barrier,

backend/src/monitor.rs

@@ -1,18 +1,43 @@
-use std::time::Duration;
+use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration};
 
 use once_cell::sync::OnceCell;
+use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
-use tokio::sync::mpsc;
+use tokio::{
+    join,
+    sync::{mpsc, RwLock},
+};
 use uuid::Uuid;
+use windmill_api::{
+    oauth2::{build_oauth_clients, OAuthClient},
+    DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT,
+};
 use windmill_common::{
     error,
+    global_settings::{
+        BASE_URL_SETTING, EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        RETENTION_PERIOD_SECS_SETTING,
+    },
     jobs::{JobKind, QueuedJob},
-    METRICS_ENABLED,
+    server::load_server_config,
+    users::truncate_token,
+    worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
+    BASE_URL, DB, METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, NPM_CONFIG_REGISTRY,
+    PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
+#[cfg(feature = "enterprise")]
+use crate::ee::verify_license_key;
+
+#[cfg(feature = "enterprise")]
+use windmill_api::LICENSE_KEY_VALID;
+
+use crate::ee::set_license_key;
+
 lazy_static::lazy_static! {
     static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
     .ok()
@@ -35,27 +60,515 @@ lazy_static::lazy_static! {
         "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
     )
     .unwrap();
+
+    static ref QUEUE_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
+        "queue_count",
+        "Number of jobs in the queue",
+        &["tag"]
+    ).unwrap();
+
+    static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
+
 }
 
-pub async fn handle_zombie_jobs_periodically<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
+pub async fn initial_load(
     db: &Pool<Postgres>,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
+    tx: tokio::sync::broadcast::Sender<()>,
+    worker_mode: bool,
+    server_mode: bool,
 ) {
-    loop {
-        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
-            _ = rx.recv() => {
-                    println!("received killpill for monitor job");
-                    break;
+    let reload_worker_config_f = async {
+        if worker_mode {
+            reload_worker_config(&db, tx, false).await;
+        }
+    };
+    let reload_custom_tags_f = async {
+        if server_mode {
+            if let Err(e) = reload_custom_tags_setting(db).await {
+                tracing::error!("Error reloading custom tags: {:?}", e)
             }
         }
+    };
+
+    let reload_base_url_f = async {
+        if let Err(e) = reload_base_url_setting(db).await {
+            tracing::error!("Error reloading base url: {:?}", e)
+        }
+    };
+
+    let reload_server_config_f = async {
+        if server_mode {
+            reload_server_config(&db).await;
+        }
+    };
+    let reload_retention_period_f = async {
+        if server_mode {
+            reload_retention_period_setting(&db).await;
+        }
+    };
+
+    let reload_request_size_f = async {
+        if server_mode {
+            reload_request_size(&db).await;
+        }
+    };
+
+    let reload_license_key_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = reload_license_key(&db).await {
+            tracing::error!("Error reloading license key: {:?}", e)
+        }
+    };
+
+    let reload_extra_pip_index_url_f = async {
+        if worker_mode {
+            reload_extra_pip_index_url_setting(&db).await;
+        }
+    };
+
+    let reload_npm_config_registry_f = async {
+        if worker_mode {
+            reload_npm_config_registry_setting(&db).await;
+        }
+    };
+
+    join!(
+        reload_worker_config_f,
+        reload_server_config_f,
+        reload_custom_tags_f,
+        reload_request_size_f,
+        reload_base_url_f,
+        reload_retention_period_f,
+        reload_license_key_f,
+        reload_extra_pip_index_url_f,
+        reload_npm_config_registry_f
+    );
+}
+
+pub async fn delete_expired_items(db: &DB) -> () {
+    let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
+        "DELETE FROM token WHERE expiration <= now()
+        RETURNING concat(substring(token for 10), '*****')",
+    )
+    .fetch_all(db)
+    .await;
+
+    match tokens_deleted_r {
+        Ok(tokens) => {
+            if tokens.len() > 0 {
+                tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
     }
+
+    let pip_resolution_r = sqlx::query_scalar!(
+        "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
+    )
+    .fetch_all(db)
+    .await;
+
+    match pip_resolution_r {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
+    }
+
+    let deleted_cache = sqlx::query_scalar!(
+            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
+        )
+        .fetch_all(db)
+        .await;
+
+    match deleted_cache {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} cache resource: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
+    }
+
+    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
+    if job_retention_secs > 0 {
+        let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                job_retention_secs
+            )
+            .fetch_all(db)
+            .await;
+
+        match deleted_jobs {
+            Ok(deleted_jobs) => {
+                if deleted_jobs.len() > 0 {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+                        deleted_jobs.len(),
+                        job_retention_secs,
+                        deleted_jobs,
+                    )
+                }
+            }
+            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+        }
+    }
+}
+
+pub async fn reload_extra_pip_index_url_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        EXTRA_PIP_INDEX_URL_SETTING,
+        "PIP_EXTRA_INDEX_URL",
+        PIP_EXTRA_INDEX_URL.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading extra_pip_index_url period: {:?}", e)
+    }
+}
+
+pub async fn reload_npm_config_registry_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        NPM_CONFIG_REGISTRY_SETTING,
+        "NPM_CONFIG_REGISTRY",
+        NPM_CONFIG_REGISTRY.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading npm_config_registry period: {:?}", e)
+    }
+}
+
+pub async fn reload_retention_period_setting(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        RETENTION_PERIOD_SECS_SETTING,
+        "JOB_RETENTION_SECS",
+        60 * 60 * 24 * 60,
+        JOB_RETENTION_SECS.clone(),
+        |x| x,
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_request_size(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        REQUEST_SIZE_LIMIT_SETTING,
+        "REQUEST_SIZE_LIMIT",
+        DEFAULT_BODY_LIMIT,
+        REQUEST_SIZE_LIMIT.clone(),
+        |x| x.mul(1024 * 1024),
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_license_key(db: &DB) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        LICENSE_KEY_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var("LICENSE_KEY")
+        .ok()
+        .and_then(|x| x.parse::<String>().ok())
+        .unwrap_or(String::new());
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting LICENSE_KEY from db config: {}",
+                truncate_token(&v)
+            );
+            value = v;
+        } else {
+            tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value);
+        }
+    };
+
+    set_license_key(value).await?;
+
+    Ok(())
+}
+
+pub async fn reload_option_string_setting(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    lock: Arc<RwLock<Option<String>>>,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var).ok();
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = Some(v)
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        if value.is_none() {
+            tracing::info!("Loaded {setting_name} setting to None");
+        }
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    default: T,
+    lock: Arc<RwLock<T>>,
+    transformer: fn(T) -> T,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var)
+        .ok()
+        .and_then(|x| x.parse::<T>().ok())
+        .unwrap_or(default);
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<T>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = transformer(v);
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+    server_mode: bool,
+) {
+    let zombie_jobs_f = async {
+        if server_mode {
+            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+        }
+    };
+    let expired_items_f = async {
+        if server_mode {
+            delete_expired_items(&db).await;
+        }
+    };
+
+    let verify_license_key_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = verify_license_key().await {
+            tracing::error!("Error verifying license key: {:?}", e);
+            let mut l = LICENSE_KEY_VALID.write().await;
+            *l = false;
+        } else {
+            let is_valid = LICENSE_KEY_VALID.read().await.clone();
+            if !is_valid {
+                let mut l = LICENSE_KEY_VALID.write().await;
+                *l = true;
+            }
+        }
+    };
+
+    let expose_queue_metrics_f = async {
+        if *METRICS_ENABLED && server_mode {
+            expose_queue_metrics(&db).await;
+        }
+    };
+    join!(
+        expired_items_f,
+        zombie_jobs_f,
+        expose_queue_metrics_f,
+        verify_license_key_f
+    );
+}
+
+pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
+    let queue_counts = sqlx::query!(
+        "SELECT tag, count(*) as count FROM queue WHERE
+        scheduled_for <= now() - ('3 seconds')::interval AND running = false
+        GROUP BY tag"
+    )
+    .fetch_all(db)
+    .await
+    .ok()
+    .unwrap_or_else(|| vec![]);
+    for q in queue_counts {
+        let count = q.count.unwrap_or(0);
+        let tag = q.tag;
+        let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+        metric.set(count as i64);
+    }
+}
+
+pub async fn reload_server_config(db: &Pool<Postgres>) {
+    let config = load_server_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading server config: {:?}", e)
+    } else {
+        let mut wc = SERVER_CONFIG.write().await;
+        tracing::info!("Reloading server config...");
+        *wc = config.unwrap()
+    }
+}
+
+pub async fn reload_worker_config(
+    db: &DB,
+    tx: tokio::sync::broadcast::Sender<()>,
+    kill_if_change: bool,
+) {
+    let config = load_worker_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading worker config: {:?}", e)
+    } else {
+        let wc = WORKER_CONFIG.read().await;
+        let config = config.unwrap();
+        if *wc != config {
+            if kill_if_change {
+                if (*wc).dedicated_worker != config.dedicated_worker {
+                    tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).init_bash != config.init_bash {
+                    tracing::info!("Init bash config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).cache_clear != config.cache_clear {
+                    tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                    tracing::info!("Waiting 5 seconds to allow others workers to start potential jobs that depend on a potential shared cache volume");
+                    tokio::time::sleep(Duration::from_secs(5)).await;
+                    if let Err(e) = windmill_worker::common::clean_cache().await {
+                        tracing::error!("Error cleaning the cache: {e}");
+                    }
+                }
+            }
+            drop(wc);
+
+            let mut wc = WORKER_CONFIG.write().await;
+            tracing::info!("Reloading worker config...");
+            *wc = config
+        }
+    }
+}
+
+pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
+    let q_base_url = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        BASE_URL_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let std_base_url = std::env::var("BASE_URL")
+        .ok()
+        .unwrap_or_else(|| "http://localhost".to_string());
+    let base_url = if let Some(q) = q_base_url {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            if v != "" {
+                v
+            } else {
+                std_base_url
+            }
+        } else {
+            tracing::error!(
+                "Could not parse base_url setting as a string, found: {:#?}",
+                &q.value
+            );
+            std_base_url
+        }
+    } else {
+        std_base_url
+    };
+
+    let q_oauth = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        OAUTH_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let oauths = if let Some(q) = q_oauth {
+        if let Ok(v) =
+            serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.value.clone())
+        {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse oauth setting as a json, found: {:#?}",
+                &q.value
+            );
+            None
+        }
+    } else {
+        None
+    };
+
+    let is_secure = base_url.starts_with("https://");
+
+    {
+        let mut l = OAUTH_CLIENTS.write().await;
+        *l = build_oauth_clients(&base_url, oauths)
+        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
+        .unwrap();
+    }
+
+    {
+        let mut l = BASE_URL.write().await;
+        *l = base_url
+    }
+
+    {
+        let mut l = IS_SECURE.write().await;
+        *l = is_secure;
+    }
+
+    Ok(())
 }
 
 async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(

backend/tests/fixtures/base.sql

@@ -17,59 +17,6 @@ INSERT INTO workspace_settings (workspace_id) VALUES
 
 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main(fail: boolean = true) {
-  if (fail) {
-    throw new Error("Failed")
-  }
-
-  return "OK"
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_script', -28028598712388162, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Error handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
-'',
-'',
-'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Recovery handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
-'',
-'',
-'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
-
-INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
-'test-workspace',
-'system',
-'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_flow'
-);
-
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 

backend/tests/fixtures/relative_bun.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'bun', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'bun', '');

backend/tests/fixtures/relative_deno.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'deno', '');

backend/tests/fixtures/relative_python.sql

@@ -0,0 +1,41 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system/same_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system_relative/different_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+from f.system.same_folder_script import main as test1
+from ..system.same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from .different_folder_script import main as test4
+
+def main():
+    return [test1(), test2(), test3(), test4()]
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'python3', '');

backend/tests/fixtures/schedule.sql

@@ -0,0 +1,52 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);

backend/tests/worker.rs

@@ -1,26 +1,81 @@
 use std::sync::Arc;
 
+#[cfg(feature = "enterprise")]
 use chrono::Timelike;
 use futures::StreamExt;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
-use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::{
-    sync::RwLock,
-    time::{timeout, Duration},
-};
-use windmill_api::jobs::{CompletedJob, Job};
+use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
+use tokio::sync::RwLock;
+
+#[cfg(feature = "enterprise")]
+use tokio::time::{timeout, Duration};
+
 use windmill_api_client::types::{
-    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+    CreateFlowBody, RawScript
 };
+
+use sqlx::query;
+
+#[cfg(feature = "enterprise")]
+use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
+
+use windmill_api_client::types::{NewScript, NewScriptLanguage};
+
+use windmill_common::worker::WORKER_CONFIG;
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
-    jobs::{JobPayload, RawCode},
-    scripts::ScriptLang,
+    jobs::{JobPayload, RawCode, JobKind},
+    scripts::{ScriptLang, ScriptHash}
 };
-use windmill_queue::{get_queued_job, PushIsolationLevel};
+use windmill_queue::PushIsolationLevel;
+use serde::Serialize;
+
+use std::str::FromStr;
+
+#[derive(Debug, sqlx::FromRow, Serialize)]
+pub struct CompletedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub started_at: chrono::DateTime<chrono::Utc>,
+    pub duration_ms: i64,
+    pub success: bool,
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    pub result: Option<serde_json::Value>,
+    pub logs: Option<String>,
+    pub deleted: bool,
+    pub raw_code: Option<String>,
+    pub canceled: bool,
+    pub canceled_by: Option<String>,
+    pub canceled_reason: Option<String>,
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    pub flow_status: Option<serde_json::Value>,
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    pub is_skipped: bool,
+    pub email: String,
+    pub visible_to_owner: bool,
+    pub mem_peak: Option<i32>,
+    pub tag: String,
+    pub script_hash: Option<ScriptHash>,
+    pub language: Option<ScriptLang>,
+    pub job_kind: JobKind,
+
+}
+
+impl CompletedJob {
+    pub fn json_result(&self) -> Option<serde_json::Value> {
+        self.result.clone()
+    }
+}
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -54,37 +109,6 @@ fn next_worker_name() -> String {
     format!("{id}/{thread_name}")
 }
 
-pub async fn get_job_by_id<'c>(
-    mut tx: Transaction<'c, Postgres>,
-    w_id: &str,
-    id: Uuid,
-) -> windmill_common::error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
-    let cjob_option = sqlx::query_as::<_, CompletedJob>(
-        "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-    )
-    .bind(id)
-    .bind(w_id)
-    .fetch_optional(&mut *tx)
-    .await?;
-    let job_option = match cjob_option {
-        Some(job) => Some(Job::CompletedJob(job)),
-        None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
-    };
-    if job_option.is_some() {
-        Ok((job_option, tx))
-    } else {
-        // check if a job had been moved in-between queries
-        let cjob_option = sqlx::query_as::<_, CompletedJob>(
-            "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-        )
-        .bind(id)
-        .bind(w_id)
-        .fetch_optional(&mut *tx)
-        .await?;
-        Ok((cjob_option.map(Job::CompletedJob), tx))
-    }
-}
-
 pub struct ApiServer {
     pub addr: std::net::SocketAddr,
     tx: tokio::sync::broadcast::Sender<()>,
@@ -115,21 +139,22 @@ impl ApiServer {
     }
 
     async fn close(self) -> anyhow::Result<()> {
+        println!("closing api server");
         let Self { tx, task, .. } = self;
         drop(tx);
         task.await.unwrap()
     }
 }
 
-async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
-    tracing::info!(
-        "{:#?}",
-        get_job_by_id(db.begin().await?, "test-workspace", id)
-            .await?
-            .0
-    );
-    Ok(())
-}
+// async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
+//     tracing::info!(
+//         "{:#?}",
+//         get_job_by_id(db.begin().await?, "test-workspace", id)
+//             .await?
+//             .0
+//     );
+//     Ok(())
+// }
 
 fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     cjob.flow_status.clone().and_then(|fs| {
@@ -299,7 +324,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!({
@@ -337,7 +362,7 @@ mod suspend_resume {
             .arg("port", json!(port))
             .run_until_complete(&db, port)
             .await
-            .result
+            .json_result()
             .unwrap();
 
         server.close().await.unwrap();
@@ -400,7 +425,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
@@ -555,7 +580,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -584,7 +609,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -626,7 +651,7 @@ def main(last, port):
             .run_until_complete(&db, server.addr.port())
             .await;
 
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(server.close().await, attempts);
         assert!(result["error"]
             .as_object()
@@ -690,7 +715,7 @@ def main(error, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await;
-        let result = cjob.result.clone().unwrap();
+        let result = cjob.json_result().clone().unwrap();
         let failed_module = get_module(&cjob, "a").unwrap();
         match failed_module {
             FlowStatusModule::Failure { .. } => {}
@@ -748,7 +773,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -757,7 +782,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!((0..257).collect::<Vec<_>>()))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
@@ -805,7 +830,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -815,7 +840,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .run_until_complete(&db, server.addr.port())
         .await;
     // println!("{:#?}", job);
-    let result = job.result.unwrap();
+    let result = job.json_result().unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
         .as_object()
@@ -855,7 +880,7 @@ impl RunJob {
             args,
             /* user */ "test-user",
             /* email  */ "test@windmill.dev",
-            /* permissioned_as */ "u/admin".to_string(),
+            /* permissioned_as */ "u/test-user".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
@@ -881,7 +906,8 @@ impl RunJob {
         let uuid = self.push(db).await;
         let listener = listen_for_completed_jobs(db).await;
         in_test_worker(db, listener.find(&uuid), port).await;
-        completed_job(uuid, db).await
+        let r = completed_job(uuid, db).await;
+        r
     }
 }
 
@@ -916,13 +942,12 @@ async fn in_test_worker<Fut: std::future::Future>(
     };
 
     /* ensure the worker quits before we return */
-    drop(quit);
+    quit.send(()).expect("send");
 
     let _: () = worker
         .await
         .expect("worker timed out")
         .expect("worker panicked");
-
     res
 }
 
@@ -946,8 +971,13 @@ fn spawn_test_worker(
     let worker_name: String = next_worker_name();
     let ip: &str = Default::default();
 
+    let tx2 = tx.clone();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
+        {
+        let mut wc = WORKER_CONFIG.write().await;
+        (*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
+        }
         windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
             &db,
             worker_instance,
@@ -956,6 +986,7 @@ fn spawn_test_worker(
             1,
             ip,
             rx,
+            tx2,
             &base_internal_url,
             None,
             Arc::new(RwLock::new(None)),
@@ -995,8 +1026,8 @@ async fn listen_for_uuid_on(
 }
 
 async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
-    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-        .bind(uuid)
+
+    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job  WHERE id = $1").bind(uuid)
         .fetch_one(db)
         .await
         .unwrap()
@@ -1106,7 +1137,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
         println!("deno flow iteration: {}", i);
         let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await;
         // println!("job: {:#?}", job.flow_status);
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i);
     }
 }
@@ -1142,7 +1173,7 @@ async fn test_identity(db: Pool<Postgres>) {
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!(42));
 }
@@ -1331,7 +1362,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
 
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(
         result,
@@ -1386,7 +1417,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
     let job = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([[42]]));
 }
@@ -1431,7 +1462,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw 123"), result);
 
@@ -1440,7 +1471,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!(-123), result);
 }
 
@@ -1489,7 +1520,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw [123]"), result);
 
@@ -1498,7 +1529,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!([-123]), result);
 }
 
@@ -1552,7 +1583,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}");
@@ -1587,7 +1618,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}");
@@ -1624,7 +1655,7 @@ func main(derp string) (string, error) {
     .arg("derp", json!("world"))
     .run_until_complete(&db, port)
     .await
-    .result
+    .json_result()
     .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1654,8 +1685,7 @@ echo "hello $msg"
     .arg("msg", json!("world"))
     .run_until_complete(&db, port)
     .await;
-
-    assert_eq!(job.result, Some(json!("hello world")));
+    assert_eq!(job.json_result(), Some(json!("hello world")));
 }
 
 #[sqlx::test(fixtures("base"))]
@@ -1682,7 +1712,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1715,7 +1745,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(3));
@@ -1747,7 +1777,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("test-workspace"));
@@ -1803,7 +1833,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(0));
@@ -1843,7 +1873,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
     let job = run_job_in_new_worker_until_complete(&db, flow, port).await;
 
     assert_eq!(
-        job.result.unwrap(),
+        job.json_result().unwrap(),
         serde_json::json!( {"error":  {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
     );
 }
@@ -1884,7 +1914,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([]));
@@ -1939,7 +1969,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(9));
@@ -2007,7 +2037,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2]));
@@ -2043,7 +2073,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 3]));
@@ -2081,7 +2111,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2118,7 +2148,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2165,7 +2195,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2202,7 +2232,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2266,7 +2296,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2, 3]));
@@ -2323,7 +2353,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     println!("{:#?}", result);
@@ -2388,7 +2418,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(0))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2404,7 +2434,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(1))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2420,7 +2450,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(2))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2436,7 +2466,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(3))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
@@ -2586,7 +2616,9 @@ async fn test_rust_client(db: Pool<Postgres>) {
     .unwrap();
 }
 
-#[sqlx::test(fixtures("base"))]
+
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_script_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2642,8 +2674,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2702,8 +2733,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2721,7 +2751,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
 }
 
 
-#[sqlx::test(fixtures("base"))]
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2778,8 +2809,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2839,8 +2869,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2856,3 +2885,203 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     )
     .await;
 }
+
+
+async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    client.create_script(
+        "test-workspace",
+        &NewScript {
+            language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
+            content: script_content,
+            path: "f/system/test_import".to_string(),
+            concurrent_limit: vec![],
+            concurrency_time_window_s: vec![],
+            cache_ttl: None,
+            dedicated_worker: None,
+            description: "".to_string(),
+            draft_only: None,
+            envs: vec![],
+            is_template: None,
+            kind: None,
+            parent_hash: None,
+            lock: vec![],
+            summary: "".to_string(),
+            tag: None,
+            schema: std::collections::HashMap::new(),
+        },
+    ).await.unwrap();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        completed.next().await; // deployed script
+
+        let script =
+            query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string())
+                .fetch_one(&db2)
+                .await
+                .unwrap();
+
+        let job = RunJob::from(JobPayload::ScriptHash {
+            path: "f/system/test_import".to_string(),
+            hash: ScriptHash(script.hash),
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+            dedicated_worker: None,
+            language,
+        }).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        let job = RunJob::from(JobPayload::Code(RawCode {
+            content: script_content,
+            path: Some("f/system/test_import".to_string()),
+            language,
+            lock: None,
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+        })).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_relative_imports_bun(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_nested_imports_bun(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_relative_imports_deno(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_nested_imports_deno(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_relative_imports_python(db: Pool<Postgres>) {
+    let content = r#"
+from f.system.same_folder_script import main as test1
+from .same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from ..system_relative.different_folder_script import main as test4
+    
+def main():
+    return [test1(), test2(), test3(), test4()]
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_nested_imports_python(db: Pool<Postgres>) {
+
+    let content = r#"
+
+from f.system_relative.nested_script import main as test
+
+def main():
+    return test()
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+}

backend/windmill-api/Cargo.toml

@@ -53,7 +53,7 @@ cron.workspace = true
 mime_guess.workspace = true
 rust-embed.workspace = true
 tracing-subscriber.workspace = true
-retainer.workspace = true
+quick_cache.workspace = true
 rand.workspace = true
 time.workspace = true
 magic-crypt.workspace = true
@@ -74,3 +74,5 @@ bytes.workspace = true
 mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
+rsa.workspace = true
+uuid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.168.2
+  version: 1.183.0
   title: Windmill API
 
   contact:
@@ -575,6 +575,81 @@ paths:
             application/json:
               schema: {}
 
+  /settings/test_smtp:
+    post:
+      summary: test smtp
+      operationId: testSmtp
+      tags:
+        - setting
+      requestBody:
+        description: test smtp payload
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                to:
+                  type: string
+                smtp:
+                  type: object
+                  properties:
+                    host:
+                      type: string
+                    username:
+                      type: string
+                    password:
+                      type: string
+                    port:
+                      type: integer
+                    from:
+                      type: string
+                    tls_implicit:
+                      type: boolean
+                  required:
+                    - host
+                    - username
+                    - password
+                    - port
+                    - from
+                    - tls_implicit
+              required:
+                - to
+                - smtp
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
+  /settings/test_license_key:
+    post:
+      summary: test license key
+      operationId: testLicenseKey
+      tags:
+        - setting
+      requestBody:
+        description: test license key
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                license_key:
+                  type: string
+              required:
+                - license_key
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -603,6 +678,45 @@ paths:
               schema:
                 type: string
 
+  /users/tutorial_progress:
+    get:
+      summary: get tutorial progress
+      operationId: getTutorialProgress
+      tags:
+        - user
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  progress:
+                    type: integer
+    post:
+      summary: update tutorial progress
+      operationId: updateTutorialProgress
+      tags:
+        - user
+      requestBody:
+        description: progress update
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                progress:
+                  type: integer
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /users/usage:
     get:
       summary: get current usage outside of premium workspaces
@@ -1040,8 +1154,12 @@ paths:
                     type: string
                   openai_resource_path:
                     type: string
+                  code_completion_enabled:
+                    type: boolean
                   error_handler:
                     type: string
+                required:
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/get_deploy_to:
     get:
@@ -1194,24 +1312,28 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/edit_openai_resource_path:
+  /w/{workspace}/workspaces/edit_copilot_config:
     post:
-      summary: edit OpenAI resource path
-      operationId: editOpenaiResourcePath
+      summary: edit copilot config
+      operationId: editCopilotConfig
       tags:
         - workspace
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: WorkspaceOpenaiResourcePath
+        description: WorkspaceCopilotConfig
         required: true
         content:
           application/json:
             schema:
               type: object
+              required:
+                - code_completion_enabled
               properties:
                 openai_resource_path:
                   type: string
+                code_completion_enabled:
+                  type: boolean
       responses:
         "200":
           description: status
@@ -1220,10 +1342,10 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/exists_openai_resource_path:
+  /w/{workspace}/workspaces/get_copilot_info:
     get:
-      summary: OpenAI resource path exists
-      operationId: existsOpenaiResourcePath
+      summary: get copilot info
+      operationId: getCopilotInfo
       tags:
         - workspace
       parameters:
@@ -1235,7 +1357,15 @@ paths:
           content:
             text/plain:
               schema:
-                type: boolean
+                type: object
+                properties:
+                  exists_openai_resource_path:
+                    type: boolean
+                  code_completion_enabled:
+                    type: boolean
+                required:
+                  - exists_openai_resource_path
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/edit_error_handler:
     post:
@@ -1934,6 +2064,12 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/Path"
+        - name: job_id
+          description: job id
+          in: query
+          schema:
+            type: string
+            format: uuid
       responses:
         "200":
           description: resource value
@@ -2004,6 +2140,58 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableResource"
 
+  /w/{workspace}/resources/list_search:
+    get:
+      summary: list resources for search
+      operationId: listSearchResource
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: resource list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
+  /w/{workspace}/resources/list_names/{name}:
+    get:
+      summary: list resource names
+      operationId: listResourceNames
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: resource list names
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    path:
+                      type: string
+                  required:
+                    - name
+                    - path
+
   /w/{workspace}/resources/type/create:
     post:
       summary: create resource_type
@@ -2138,6 +2326,65 @@ paths:
                 items:
                   type: string
 
+  /resources/type/hub/list:
+    get:
+      summary: list hub resource types
+      operationId: listHubResourceTypes
+      tags:
+        - resource
+      responses:
+        "200":
+          description: resource type details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                    name:
+                      type: string
+                    schema: {}
+                  required:
+                    - id
+                    - name
+
+  /resources/type/hub/query:
+    get:
+      summary: query hub resource types by similarity
+      operationId: queryHubResourceTypes
+      tags:
+        - resource
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: resource type details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
   /scripts/hub/list:
     get:
       summary: list all available hub scripts
@@ -2359,6 +2606,72 @@ paths:
                   - content
                   - language
 
+  /scripts/hub/query:
+    get:
+      summary: query hub scripts by similarity
+      operationId: queryHubScripts
+      tags:
+        - script
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: kind
+          description: query scripts kind
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
+  /w/{workspace}/scripts/list_search:
+    get:
+      summary: list scripts for search
+      operationId: listSearchScript
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: script list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    content:
+                      type: string
+                  required:
+                    - path
+                    - content
+
   /w/{workspace}/scripts/list:
     get:
       summary: list all available scripts
@@ -3004,6 +3317,31 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/flows/list_search:
+    get:
+      summary: list flows for search
+      operationId: listSearchFlow
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: flow list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/flows/list:
     get:
       summary: list all available flows
@@ -3307,6 +3645,31 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/apps/list_search:
+    get:
+      summary: list apps for search
+      operationId: listSearchApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: app list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -3452,6 +3815,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
+  /w/{workspace}/apps_u/public_resource/{path}:
+    get:
+      summary: get public resource
+      operationId: get public resource
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -3887,6 +4266,27 @@ paths:
                 required:
                   - database_length
 
+  /w/{workspace}/jobs/completed/count:
+    get:
+      summary: get completed count
+      operationId: getCompletedCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: completed count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
   /w/{workspace}/jobs/queue/cancel_all:
     post:
       summary: cancel all jobs
@@ -3964,6 +4364,7 @@ paths:
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/CreatedOrStartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
@@ -5164,6 +5565,80 @@ paths:
                 items:
                   $ref: "#/components/schemas/WorkerPing"
 
+  /configs/list_worker_groups:
+    get:
+      summary: list worker groups
+      operationId: listWorkerGroups
+      tags:
+        - config
+      responses:
+        "200":
+          description: a list of worker group configs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    config: {}
+                  required:
+                    - name
+                    - config
+
+  /configs/get/{name}:
+    get:
+      summary: get config
+      operationId: get config
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: a config
+          content:
+            application/json:
+              schema: {}
+
+  /configs/update/{name}:
+    post:
+      summary: Update config
+      operationId: updateConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      requestBody:
+        description: worker group
+        required: true
+        content:
+          application/json:
+            schema: {}
+      responses:
+        "200":
+          description: Update a worker group
+          content:
+            text/plain:
+              schema:
+                type: string
+    delete:
+      summary: Delete Config
+      operationId: deleteConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: Delete config
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/acls/get/{kind}/{path}:
     get:
       summary: get granular acls
@@ -6589,7 +7064,7 @@ components:
             - "workspaces.edit_deploy_to"
             - "workspaces.edit_auto_invite_domain"
             - "workspaces.edit_webhook"
-            - "workspaces.edit_openai_resource_path"
+            - "workspaces.edit_copilot_config"
             - "workspaces.edit_error_handler"
             - "workspaces.create"
             - "workspaces.update"
@@ -7084,6 +7559,8 @@ components:
           type: array
           items:
             type: string
+        worker_group:
+          type: string
       required:
         - worker
         - worker_instance
@@ -7091,6 +7568,7 @@ components:
         - started_at
         - ip
         - jobs_executed
+        - worker_group
 
     UserWorkspaceList:
       type: object

backend/windmill-api/src/apps.rs

@@ -15,7 +15,9 @@ use crate::{
     HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Json, Path, Query},
+    response::IntoResponse,
     routing::{delete, get, post},
     Router,
 };
@@ -35,7 +37,7 @@ use windmill_common::{
     jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
+        http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
     },
 };
 use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
@@ -43,6 +45,7 @@ use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
+        .route("/list_search", get(list_search_apps))
         .route("/get/p/*path", get(get_app))
         .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
@@ -57,6 +60,7 @@ pub fn unauthed_service() -> Router {
     Router::new()
         .route("/execute_component/*path", post(execute_component))
         .route("/public_app/:secret", get(get_public_app_by_secret))
+        .route("/public_resource/*path", get(get_public_resource))
 }
 
 pub fn global_service() -> Router {
@@ -159,6 +163,38 @@ pub struct EditApp {
     pub policy: Option<Policy>,
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchApp {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_apps(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchApp>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchApp,
+        "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_apps(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -341,6 +377,27 @@ async fn get_public_app_by_secret(
     Ok(Json(app))
 }
 
+async fn get_public_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    if !path.starts_with("f/app_themes/") {
+        return Err(Error::BadRequest(
+            "Only app themes are public resources".to_string(),
+        ));
+    }
+    let res = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten();
+    Ok(Json(res))
+}
+
 async fn get_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -489,14 +546,19 @@ async fn create_app(
     Ok((StatusCode::CREATED, app.path))
 }
 
-async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchUiData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 pub async fn get_hub_app_by_id(
@@ -508,6 +570,7 @@ pub async fn get_hub_app_by_id(
         &format!("https://hub.windmill.dev/apps/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()

backend/windmill-api/src/capture.rs

@@ -9,7 +9,7 @@
 use axum::{
     extract::{Extension, Path, Query},
     routing::{get, post, put},
-    Json, Router,
+    Router,
 };
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
@@ -21,7 +21,7 @@ use windmill_common::{
 
 use crate::{
     db::{ApiAuthed, DB},
-    jobs::add_include_headers,
+    jobs::{add_include_headers, add_raw_string, JsonOrForm},
 };
 
 const KEEP_LAST: i64 = 8;
@@ -95,11 +95,12 @@ pub async fn update_payload(
     Path((w_id, path)): Path<(String, StripPath)>,
     Query(run_query): Query<IncludeHeaderQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
-
     let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
+
     sqlx::query!(
         "
        UPDATE capture

backend/windmill-api/src/configs.rs

@@ -0,0 +1,109 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use axum::{
+    extract::{Extension, Path},
+    routing::{get, post},
+    Json, Router,
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use windmill_common::{
+    db::UserDB,
+    error::{self},
+    DB,
+};
+
+use crate::{db::ApiAuthed, utils::require_super_admin};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/list_worker_groups", get(list_worker_groups))
+        .route("/update/:name", post(update_config).delete(delete_config))
+        .route("/get/:name", get(get_config))
+}
+
+#[derive(Serialize, Deserialize, FromRow)]
+struct Config {
+    name: String,
+    config: serde_json::Value,
+}
+
+async fn list_worker_groups(Extension(db): Extension<DB>) -> error::JsonResult<Vec<Config>> {
+    let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+        .fetch_all(&db)
+        .await?;
+    Ok(Json(rows))
+}
+
+async fn get_config(
+    authed: ApiAuthed,
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Option<serde_json::Value>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
+        .fetch_optional(&db)
+        .await?
+        .map(|c| c.config);
+
+    Ok(Json(config))
+}
+
+async fn update_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(config): Json<serde_json::Value>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    #[cfg(not(feature = "enterprise"))]
+    if name.starts_with("worker__") {
+        return Err(error::Error::BadRequest(
+            "Worker groups configurable from UI available only in the enterprise version"
+                .to_string(),
+        ));
+    }
+
+    sqlx::query!(
+        "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+        &name,
+        config
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Updated config {name}"))
+}
+
+async fn delete_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    let tx = user_db.begin(&authed).await?;
+
+    require_super_admin(&db, &authed.email).await?;
+    tx.commit().await?;
+
+    let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name)
+        .fetch_all(&db)
+        .await?;
+
+    if deleted.len() == 0 {
+        return Err(error::Error::NotFound(format!(
+            "Config {name} not found",
+            name = name
+        )));
+    }
+    Ok(format!("Deleted config {name}"))
+}

backend/windmill-api/src/ee.rs

@@ -0,0 +1,41 @@
+use anyhow::anyhow;
+use base64::Engine;
+use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
+use sha2::Sha256;
+
+pub async fn validate_license_key(license_key: String) -> anyhow::Result<String> {
+    let mut splitted_lk = license_key.split(".");
+    if splitted_lk.clone().count() != 3 {
+        return Err(anyhow!(
+            "license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)"
+        ));
+    }
+
+    let id = splitted_lk.next().unwrap();
+    let expiry = splitted_lk.next().unwrap();
+    let signature_b64 = splitted_lk.next().unwrap();
+
+    const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
+    let pub_key = rsa::RsaPublicKey::from_public_key_der(
+        &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
+    )?;
+    let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
+    rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
+        .verify(
+            &format!("{id}{expiry}").as_bytes(),
+            &rsa::pss::Signature::from(signature),
+        )
+        .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
+
+    let expiry_nb = expiry.parse::<u64>()?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        return Err(anyhow!("License key expired".to_string()));
+    };
+
+    Ok(id.to_string())
+}

backend/windmill-api/src/flows.rs

@@ -14,6 +14,8 @@ use crate::{
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
+use axum::body::StreamBody;
+use axum::response::IntoResponse;
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
@@ -24,8 +26,9 @@ use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
-use sqlx::{Postgres, Transaction};
+use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::utils::query_elems_from_hub;
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
@@ -33,15 +36,14 @@ use windmill_common::{
     jobs::JobPayload,
     schedule::Schedule,
     scripts::Schema,
-    utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
-    },
+    utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath},
 };
 use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_flows))
+        .route("/list_search", get(list_search_flows))
         .route("/create", post(create_flow))
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
@@ -58,6 +60,38 @@ pub fn global_service() -> Router {
         .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchFlow {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_flows(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchFlow>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchFlow,
+        "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_flows(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -123,14 +157,19 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchFlowData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 async fn list_paths(
@@ -160,6 +199,7 @@ pub async fn get_hub_flow_by_id(
         &format!("https://hub.windmill.dev/flows/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()