Merge branch 'main' into instructions_helm

* chore(main): release 1.177.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -1,19 +1,12 @@
-DB_PASSWORD=changeme
 DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
 
 WM_IMAGE=ghcr.io/windmill-labs/windmill:main
 WM_LICENSE_KEY=""
 
 # For Enterprise Edition, comment the 2 lines above and uncomment below
-# WINDMILL_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
+# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
 # WM_LICENSE_KEY="<id>.<expiry>.<signature>"
 
 
-# this is the url that your instance is publicly exposed to
-WM_BASE_URL=http://localhost
-
 # To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
 # To have caddy take care of automatic TLS
-
-
-WM_REQUEST_SIZE_LIMIT=50097152 # 50MB (The size limit for any requests, including script/flow args inputs)

.github/DockerfileBackendTests

@@ -37,9 +37,10 @@ RUN apt-get update \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr nodejs npm\
+    libv8-dev tesseract-ocr nodejs npm clang libclang-dev\
     && rm -rf /var/lib/apt/lists/*
 
+
 RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
@@ -57,7 +58,7 @@ RUN /usr/local/bin/python3 -m pip install nltk
 RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
+COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
 RUN apt-get update \

.github/change-versions.sh

@@ -5,7 +5,7 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json

.github/workflows/backend-test.yml

@@ -42,4 +42,4 @@ jobs:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
           DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
-          test --all -- --nocapture
+          test --features enterprise --all -- --nocapture 

.github/workflows/benchmark.yml

@@ -19,15 +19,28 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
       windmill:
-        image: ghcr.io/windmill-labs/windmill:main
+        image: ghcr.io/windmill-labs/windmill-ee:main
         env:
           DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow
         options: >-
           --pull always --health-interval 10s --health-timeout 5s
           --health-retries 5 --health-cmd "curl
           http://localhost:8000/api/version"
         ports:
           - 8000:8000
+      windmill-worker:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          DISABLE_SERVER: true
+          WORKER_GROUP: dedicated
+          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+        options: >-
+          --pull always
     steps:
       - uses: denoland/setup-deno@v1
         with:
@@ -37,13 +50,9 @@ jobs:
           ref: benchmarks
       - name: benchmark
         timeout-minutes: 10
-        run: deno run --unstable -A
+        run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          --host http://localhost:8000
-          -e admin@windmill.dev
-          -p changeme
           -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
-          --branch ${GITHUB_REF##ref/head/}
       - name: Push changes
         run: |
           pwd

CHANGELOG.md

@@ -1,6 +1,160 @@
 # Changelog
 
 
+## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
+
+
+### Features
+
+* add custom oauth support ([#2336](https://github.com/windmill-labs/windmill/issues/2336)) ([01277f4](https://github.com/windmill-labs/windmill/commit/01277f4d3b8bb04b955d5bbb2ed69c1c7c8f4f9e))
+* support automatic reconnection to pg ([ccaa05d](https://github.com/windmill-labs/windmill/commit/ccaa05d4bf5954c3fb8678239d2962cac6550a5a))
+
+
+### Bug Fixes
+
+* fix resource type picker object reinitialization ([f0f15c4](https://github.com/windmill-labs/windmill/commit/f0f15c47cb35cc1e3cfa13549465803a1e970770))
+* **frontend:** Fix build ([#2330](https://github.com/windmill-labs/windmill/issues/2330)) ([46592af](https://github.com/windmill-labs/windmill/commit/46592affd3d51b54632a2a7a281c11141edcb4a5))
+* **frontend:** Fix markdown dark mode ([#2329](https://github.com/windmill-labs/windmill/issues/2329)) ([6c19740](https://github.com/windmill-labs/windmill/commit/6c197407185810f43c47d4107007bd69814a1d65))
+* set min size of components to 1 ([d298093](https://github.com/windmill-labs/windmill/commit/d298093e29bd9983c7631a8f8c80e47b768bb93c))
+
+## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24)
+
+
+### Features
+
+* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254))
+* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6))
+* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33))
+* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85))
+* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8))
+
+
+### Bug Fixes
+
+* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0))
+* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64))
+* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1))
+* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9))
+* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781))
+* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55))
+* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9))
+* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91))
+
+## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19)
+
+
+### Features
+
+* add batch jobs ([#2306](https://github.com/windmill-labs/windmill/issues/2306)) ([5867e5d](https://github.com/windmill-labs/windmill/commit/5867e5d0f80fd515fab165659831b5ee9a8c3f97))
+* add dediacted worker env var ([#2296](https://github.com/windmill-labs/windmill/issues/2296)) ([e0c6eee](https://github.com/windmill-labs/windmill/commit/e0c6eee16e535b3a7d803a7978e463404f5fec30))
+* dedicated benchmarks ([#2297](https://github.com/windmill-labs/windmill/issues/2297)) ([c549239](https://github.com/windmill-labs/windmill/commit/c5492396843ddd9143ffe890696d0317c970de36))
+* **frontend:** Add component control doc ([#2295](https://github.com/windmill-labs/windmill/issues/2295)) ([26f8863](https://github.com/windmill-labs/windmill/commit/26f88636f0b972d4fe4931ed02135c38b27a56d2))
+* suggest adding openai key on workspace creation ([a6b3b2f](https://github.com/windmill-labs/windmill/commit/a6b3b2f63b317825a3d80218cbb606b9f610c221))
+* support pinned versions for bun in deployed scripts ([03806dc](https://github.com/windmill-labs/windmill/commit/03806dc3907cba724be14acb6aadf5be6e35cdb6))
+
+
+### Bug Fixes
+
+* add HOME to bun and deno ([0e3ecc7](https://github.com/windmill-labs/windmill/commit/0e3ecc7d6025c173135f20bacc33a0dc972ec222))
+* add queue_count to metrics ([9ced883](https://github.com/windmill-labs/windmill/commit/9ced8834a45151c6900b1eb33eca2cff4886a065))
+* ai improve prompts ([#2310](https://github.com/windmill-labs/windmill/issues/2310)) ([b647213](https://github.com/windmill-labs/windmill/commit/b647213b2c968b0cb1f90c97d94e8023c415dd55))
+* **frontend:** add missing key ([#2299](https://github.com/windmill-labs/windmill/issues/2299)) ([39d2467](https://github.com/windmill-labs/windmill/commit/39d24672ddd696372e55e9b4566f322a322385a8))
+* **frontend:** Always mount components ([#2309](https://github.com/windmill-labs/windmill/issues/2309)) ([34f94aa](https://github.com/windmill-labs/windmill/commit/34f94aa50e92254114c046fa8b7e900d93807937))
+* **frontend:** fix alignment ([#2307](https://github.com/windmill-labs/windmill/issues/2307)) ([f9fc6f1](https://github.com/windmill-labs/windmill/commit/f9fc6f19482e68c9ccba0014879fd8761662c36a))
+* **frontend:** Fix rich result styling + add title and hideDetails config ([#2294](https://github.com/windmill-labs/windmill/issues/2294)) ([732daef](https://github.com/windmill-labs/windmill/commit/732daef1c3515f7df3e09deac691bb585f9859cd))
+* **frontend:** fix tab styling + component bg ([#2308](https://github.com/windmill-labs/windmill/issues/2308)) ([5e773d3](https://github.com/windmill-labs/windmill/commit/5e773d386343f003425173207c166e3c4eeef956))
+* **frontend:** fix theme make default ([#2304](https://github.com/windmill-labs/windmill/issues/2304)) ([4629819](https://github.com/windmill-labs/windmill/commit/46298197c5333a81b9b8a004027ab9a856bdada4))
+* **frontend:** fix theme UI ([#2305](https://github.com/windmill-labs/windmill/issues/2305)) ([576f76b](https://github.com/windmill-labs/windmill/commit/576f76b1ffe9c50c8ccaca8c5e34d0ec03aebf3f))
+* validate more strongly usernames ([47094bb](https://github.com/windmill-labs/windmill/commit/47094bb8d1c6f4ba621d42515dede061fd04afdd))
+
+## [1.174.0](https://github.com/windmill-labs/windmill/compare/v1.173.0...v1.174.0) (2023-09-15)
+
+
+### Features
+
+* ai gen support all langs ([#2276](https://github.com/windmill-labs/windmill/issues/2276)) ([39590b3](https://github.com/windmill-labs/windmill/commit/39590b3d2592b2d08117c0f70829c13f1efb4885))
+* bun absolute/relative imports + tests ([#2286](https://github.com/windmill-labs/windmill/issues/2286)) ([e5ce85b](https://github.com/windmill-labs/windmill/commit/e5ce85b9affe665342f24b1d39ce3d03db09b941))
+* **frontend:** Global CSS editor ([#2178](https://github.com/windmill-labs/windmill/issues/2178)) ([7e9ee39](https://github.com/windmill-labs/windmill/commit/7e9ee39aa69bc31766b5e4f4aab498c8f14067cd))
+
+## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
+
+
+### Features
+
+* cli sync on windows ([#2283](https://github.com/windmill-labs/windmill/issues/2283)) ([c371cb3](https://github.com/windmill-labs/windmill/commit/c371cb397ab3d0c534e2c553d1dfb1ad5176d2a6))
+
+
+### Bug Fixes
+
+* accept jobs whose duration > 24 days ([2c00894](https://github.com/windmill-labs/windmill/commit/2c00894122aa8caee59b20625935284de6902950))
+
+## [1.172.1](https://github.com/windmill-labs/windmill/compare/v1.172.0...v1.172.1) (2023-09-14)
+
+
+### Bug Fixes
+
+* improve splitpane + improve deleting conditional tab ([1629008](https://github.com/windmill-labs/windmill/commit/1629008eb2eb48ff9cc2cf6b3a351efcf682244d))
+* update to svelte 4 ([#2280](https://github.com/windmill-labs/windmill/issues/2280)) ([90c10d8](https://github.com/windmill-labs/windmill/commit/90c10d803b4c47a9e1ac5b9e49e2a614344299a9))
+
+## [1.172.0](https://github.com/windmill-labs/windmill/compare/v1.171.0...v1.172.0) (2023-09-13)
+
+
+### Features
+
+* improve ai flow ([#2270](https://github.com/windmill-labs/windmill/issues/2270)) ([b23417a](https://github.com/windmill-labs/windmill/commit/b23417ab5b9938bbdf9db6449102760ff8c80152))
+* worker groups admin panel ([#2277](https://github.com/windmill-labs/windmill/issues/2277)) ([070b162](https://github.com/windmill-labs/windmill/commit/070b16222bc666866284180b3878f4d4f27bfa85))
+
+
+### Bug Fixes
+
+* ai flow nits ([#2272](https://github.com/windmill-labs/windmill/issues/2272)) ([8f6f46d](https://github.com/windmill-labs/windmill/commit/8f6f46de199d58133b9faa77cdbcbcfd6cb962f7))
+
+## [1.171.0](https://github.com/windmill-labs/windmill/compare/v1.170.0...v1.171.0) (2023-09-12)
+
+
+### Features
+
+* attempt to SIGTERM before SIGKILL for bash ([f40bbba](https://github.com/windmill-labs/windmill/commit/f40bbba519a97cbb1ec142c335f038dbebcd4e7c))
+* zero copy result for job result ([#2263](https://github.com/windmill-labs/windmill/issues/2263)) ([22a7da5](https://github.com/windmill-labs/windmill/commit/22a7da58b1d20721892906cba2dee6fbeb1cc1fd))
+
+
+### Bug Fixes
+
+* 2257 TIME  convertion in pg_executor.rs ([#2267](https://github.com/windmill-labs/windmill/issues/2267)) ([3d71253](https://github.com/windmill-labs/windmill/commit/3d71253abdb0dff1670a796d07a53ecd0a98414e))
+* fix field duplicate in app background settings ([164cdaf](https://github.com/windmill-labs/windmill/commit/164cdaf09464646dee4e70a699222a454eb0d898))
+* improve bun lockfile resolution ([9103ec4](https://github.com/windmill-labs/windmill/commit/9103ec445db81395a5851202eecb87301d0b4987))
+* remove result and args from list completed and list queue jobs ([e7e63e1](https://github.com/windmill-labs/windmill/commit/e7e63e111a73e0986050a8fe7fdc18784ba902b0))
+
+## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
+
+
+### Features
+
+* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
+* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
+
+
+### Bug Fixes
+
+* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
+
+## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
+
+
+### Features
+
+* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
+* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
+* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
+* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
+* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
+
+
+### Bug Fixes
+
+* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
+* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
+
 ## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
 
 

Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl nodejs npm
+    curl nodejs
 
 RUN rustup component add rustfmt
 
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.2 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

LICENSE

@@ -1,16 +1,22 @@
-
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
+./LICENSE-AGPL)
 
-Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
-Every file is under License AGPL unless otherwise specified 
-or belonging to one of the below cases:
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
+specified. Every file is under License AGPL unless otherwise specified or
+belonging to one of the below cases:
 
-The files under backend/ are AGPLv3 Licensed.
-The files under frontend/ are AGPLv3 Licensed.
-The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+The files under backend/ are AGPLv3 Licensed, except any snippets of code under
+the compile flag "enterprise". Those snippets and files are under a proprietary
+and commercial license. The files under frontend/ are AGPLv3 Licensed, except
+any snippets of code that require a positive license check to be activated.
+Those snippets and files are under a proprietary and commercial license. Private
+and public forks MUST not include any of the above proprietary and commercial
+code. Windmill Labs, Inc. provide tools to clean the codebase from those
+snippets upon demand. The files under python-client/ deno-client/ go-client/ are
+Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
 
-All third party components incorporated into the Windmill Software are licensed under the
-original license provided by the owner of the applicable component.
+All third party components incorporated into the Windmill Software are licensed
+under the original license provided by the owner of the applicable component.

README.md

@@ -211,7 +211,6 @@ compiling from source or using without a postgres super user, see
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/oauth.json -o oauth.json
 
 docker compose up -d
 ```
@@ -377,6 +376,7 @@ it being synced automatically everyday.
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path>                                                                                                            | Worker                |
 | CUSTOM_TAGS                                   | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
 | JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
 | WAIT_RESULT_FAST_POLL_INTERVAL_MS             | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
@@ -395,7 +395,7 @@ it being synced automatically everyday.
 | SMTP_USERNAME                                 | None                                       | username for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
-| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | false                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
+| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
 | GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |

backend/.sqlx/query-03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
+}

backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM config WHERE name = $1 RETURNING name",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a"
+}

backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json

@@ -37,7 +37,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT script_path FROM completed_job  WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "script_path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd"
+}

backend/.sqlx/query-240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -37,6 +37,11 @@
         "ordinal": 6,
         "name": "custom_tags",
         "type_info": "TextArray"
+      },
+      {
+        "ordinal": 7,
+        "name": "worker_group",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -52,8 +57,9 @@
       false,
       false,
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "4f6b3b472b4b78c0325cf3755f9ef1806d2e82328ceccbeade8cc2333c6dfe47"
+  "hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
 }

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration_ms",
-        "type_info": "Int4"
+        "type_info": "Int8"
       }
     ],
     "parameters": {
@@ -67,7 +67,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, count(*) as count FROM queue WHERE\n        scheduled_for <= now() - ('3 seconds')::interval AND running = false\n        GROUP BY tag",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a"
+}

backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json

@@ -28,7 +28,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Int4",
+        "TextArray",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32"
+  "hash": "47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f"
 }

backend/.sqlx/query-502781c4e2fc692db7a66b850450a1934b2403a5cff0421fe9609f9b99d8ee95.json

@@ -11,7 +11,7 @@
     ],
     "parameters": {
       "Left": [
-        "Int4"
+        "Int8"
       ]
     },
     "nullable": [

backend/.sqlx/query-584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a"
+}

backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "TextArray"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf"
-}

backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json

@@ -0,0 +1,73 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH uuid_table as (\n            select gen_random_uuid() as uuid from generate_series(1, $11)\n        )\n        INSERT INTO queue \n            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n            (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n        RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d"
+}

backend/.sqlx/query-8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM password WHERE login_type != 'password'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483"
+}

backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = 'server'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+}

backend/.sqlx/query-8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM resource_type WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6"
+}

backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "avg_duration_s",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1"
-}

backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json

@@ -60,7 +60,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47"
+}

backend/.sqlx/query-a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "avg_duration_s",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0"
+}

backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49"
+}

backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429"
+}

backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name LIKE 'worker__%'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7"
+}

backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9"
+}

backend/.sqlx/query-d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4"
+}

backend/.sqlx/query-e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468"
+}

backend/.sqlx/query-ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
+}

backend/.sqlx/query-f516ca558816c2cbab3c8ae865ef8f764aa4686e7df9c53d30d49a3dbbf36af4.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration",
-        "type_info": "Int8"
+        "type_info": "Numeric"
       }
     ],
     "parameters": {

backend/.sqlx/query-f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "database_length!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.168.3"
+version = "1.177.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.168.3"
+version = "1.177.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -54,7 +54,6 @@ tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
 git-version.workspace = true
-rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
@@ -64,6 +63,8 @@ once_cell.workspace = true
 prometheus.workspace = true
 uuid.workspace = true
 gethostname.workspace = true
+serde_json.workspace = true
+serde.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -96,7 +97,7 @@ tower = "^0"
 tower-http = { version = "^0", features = ["trace", "cors"] }
 tower-cookies = "^0"
 serde = "^1"
-serde_json = { version = "^1", features = ["preserve_order"] }
+serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
@@ -112,7 +113,7 @@ mime_guess = "^2"
 hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
-retainer = "^0"
+quick_cache = "^0"
 rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
@@ -155,7 +156,9 @@ sqlx = { version = "^0", features = [
     "chrono",
     "postgres",
     "runtime-tokio-rustls",
+    "bigdecimal"
 ] }
+bigdecimal = "^0"
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
@@ -187,6 +190,7 @@ postgres-native-tls = "^0"
 native-tls = "^0"
 samael = { version = "0.0.12", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
-rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
+rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
 pem = "3.0.1"
+nix = { version = "0.27.1", features = ["process", "signal"] }

backend/migrations/20230908155455_add_worker_group.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE worker_group_config;
+
+ALTER TABLE worker_ping DROP COLUMN worker_group;

backend/migrations/20230908155455_add_worker_group.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE worker_group_config (
+    name VARCHAR(255) PRIMARY KEY,
+    config JSONB DEFAULT '{}'::jsonb
+);
+
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS worker_group VARCHAR(255) NOT NULL DEFAULT 'default';
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS dedicated_worker VARCHAR(255);

backend/migrations/20230914062021_fix_group_grant_roles.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914062021_fix_group_grant_roles.up.sql

@@ -0,0 +1 @@
+-- Add up migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE completed_job ALTER COLUMN duration_ms TYPE bigint;
+ALTER TABLE completed_job ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN canceled_by TYPE VARCHAR(255);

backend/migrations/20230915084031_add_themes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230915084031_add_themes.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;
+INSERT INTO resource (workspace_id, path, value, description, resource_type) SELECT id, 'f/app_themes/theme_0', '{"name": "Default Theme", "value": ""}', 'The default app theme', 'app_theme' FROM workspace ON CONFLICT DO NOTHING

backend/migrations/20230917075933_rename_config.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE IF exists config RENAME TO worker_group_config ;

backend/migrations/20230917075933_rename_config.up.sql

@@ -0,0 +1,29 @@
+-- Add up migration script here
+ALTER TABLE IF exists worker_group_config RENAME TO config;
+UPDATE config SET name = 'worker__' || name;
+
+CREATE FUNCTION "notify_config_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_config_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_config_change"
+ AFTER INSERT OR UPDATE ON "config"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_config_change" ();
+
+CREATE FUNCTION "notify_global_setting_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_global_setting_change"
+ AFTER INSERT OR UPDATE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_change" ();

backend/migrations/20230921084441_add_default_worker_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230921084441_add_default_worker_groups.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+INSERT INTO config (name, config) VALUES  
+    ('worker__default', '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun"]}'), 
+    ('worker__native', '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql", "snowflake"]}') ON CONFLICT DO NOTHING;

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -35,6 +35,9 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "dateutil" => "python-dateutil",
     "mailparser" => "mail-parser",
     "mailparser-reply" => "mail-parser-reply",
+    "gitlab" => "python-gitlab",
+    "smbclient" => "smbprotocol",
+    "playhouse" => "peewee",
 };
 
 fn replace_import(x: String) -> String {

backend/src/ee.rs

@@ -1,41 +1,49 @@
 #[cfg(feature = "enterprise")]
-use base64::Engine;
-#[cfg(feature = "enterprise")]
-use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
-#[cfg(feature = "enterprise")]
-use sha2::Sha256;
+use windmill_common::error;
 
-#[cfg(feature = "enterprise")]
-pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
-    if let Some(license_key) = license_key {
-        let mut splitted_lk = license_key.split(".");
-        if splitted_lk.clone().count() != 3 {
-            panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
-        }
-        let id = splitted_lk.next().unwrap();
-        let expiry = splitted_lk.next().unwrap();
-        let signature_b64 = splitted_lk.next().unwrap();
+pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
+    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
 
-        let expiry_nb = expiry.parse::<u64>()?;
-        if expiry_nb < chrono::Utc::now().timestamp() as u64 {
-            panic!(
-                "License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
-                chrono::Utc::now().timestamp()
-            );
-        }
-        const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
-        let pub_key = rsa::RsaPublicKey::from_public_key_der(
-            &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
-        )?;
-        let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
-        rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
-            .verify(
-                &format!("{id}{expiry}").as_bytes(),
-                &rsa::pss::Signature::from(signature),
-            )
-            .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
-    } else {
-        panic!("License key is required for the enterprise edition");
+    let id = validate_license_key(license_key.clone()).await?;
+    {
+        let mut l = LICENSE_KEY_ID.write().await;
+        *l = id.to_string()
     }
+
+    {
+        let mut l = LICENSE_KEY.write().await;
+        *l = license_key
+    }
+    {
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = true
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn verify_license_key() -> error::Result<()> {
+    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::error::to_anyhow;
+
+    let expiry_nb = LICENSE_KEY
+        .read()
+        .await
+        .clone()
+        .split(".")
+        .nth(1)
+        .unwrap_or_else(|| "")
+        .parse::<u64>()
+        .map_err(to_anyhow)?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = false;
+    };
     Ok(())
 }

backend/src/main.rs

@@ -8,19 +8,26 @@
 
 use gethostname::gethostname;
 use git_version::git_version;
-use monitor::handle_zombie_jobs_periodically;
-use sqlx::{Pool, Postgres};
+use rand::Rng;
+use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     sync::Arc,
+    time::Duration,
 };
 use tokio::{
     fs::{metadata, DirBuilder},
-    join,
     sync::RwLock,
 };
-use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
+use windmill_common::{
+    global_settings::{
+        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, LICENSE_KEY_SETTING, OAUTH_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
+    },
+    utils::rd_string,
+    worker::{reload_custom_tags_setting, WORKER_GROUP},
+    DB, METRICS_ADDR,
+};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -28,6 +35,11 @@ use windmill_worker::{
     PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
 
+use crate::monitor::{
+    initial_load, monitor_db, reload_base_url_setting, reload_license_key,
+    reload_retention_period_setting, reload_server_config, reload_worker_config,
+};
+
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_PORT: u16 = 8000;
@@ -126,16 +138,9 @@ Windmill Community Edition {GIT_VERSION}
 
     display_config(&ENV_SETTINGS);
 
-    tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
-    if let Some(ref smtp) = *SMTP_CLIENT {
-        tracing::info!("Smtp client defined. Testing connection...");
-        if let Err(e) = smtp.connect().await {
-            tracing::error!("Failed to connect to smtp server: {}", e);
-        } else {
-            tracing::info!("Smtp client connected.");
-        }
-    }
-    if server_mode || num_workers > 0 {
+    let worker_mode = num_workers > 0;
+
+    if server_mode || worker_mode {
         let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
 
         let port = if server_mode {
@@ -144,6 +149,13 @@ Windmill Community Edition {GIT_VERSION}
             port_var.unwrap_or(0)
         };
 
+        // since it's only on server mode, the port is statically defined
+        let base_internal_url: String = format!("http://localhost:{}", port.to_string());
+
+        initial_load(&db, tx.clone(), worker_mode, server_mode).await;
+
+        monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
+
         if std::env::var("BASE_INTERNAL_URL").is_ok() {
             tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
         }
@@ -161,10 +173,11 @@ Windmill Community Edition {GIT_VERSION}
         let workers_f = async {
             let port = port_rx.await?;
             let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-            if num_workers > 0 {
+            if worker_mode {
                 run_workers(
                     db.clone(),
                     rx.resubscribe(),
+                    tx.clone(),
                     num_workers,
                     base_internal_url.clone(),
                     rsmq.clone(),
@@ -176,29 +189,130 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
-        let rsmq2 = rsmq.clone();
         let monitor_f = async {
-            if server_mode {
-                // since it's only on server mode, the port is statically defined
-                let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-                monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2).await;
-            }
+            let db = db.clone();
+            let tx = tx.clone();
+            let rsmq = rsmq.clone();
+
+            let mut rx = rx.resubscribe();
+            let base_internal_url = base_internal_url.to_string();
+            let rd_delay = rand::thread_rng().gen_range(0..30);
+            tokio::spawn(async move {
+                //monitor_db is applied at start, no need to apply it twice
+                tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+
+                let mut listener = retry_listen_pg(&db).await;
+
+                loop {
+                    tokio::select! {
+                        _ = tokio::time::sleep(Duration::from_secs(30))    => {
+                            monitor_db(
+                                &db,
+                                &base_internal_url,
+                                rsmq.clone(),
+                                server_mode,
+                            )
+                            .await;
+                        },
+                        notification = listener.recv() => {
+                            match notification {
+                                Ok(n) => {
+                                    tracing::info!("Received new pg notification: {n:?}");
+                                    match n.channel() {
+                                        "notify_config_change" => {
+                                            tracing::info!("Config change detected");
+                                            match n.payload() {
+                                                "server" if server_mode => {
+                                                    tracing::info!("Server config change detected");
+                                                    reload_server_config(&db).await;
+                                                },
+                                                a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
+                                                    tracing::info!("Worker config change detected");
+                                                    reload_worker_config(&db, tx.clone(), true).await;
+                                                },
+                                                _ => {
+                                                    ()
+                                                }
+                                            }
+                                        },
+                                        "notify_global_setting_change" => {
+                                            tracing::info!("Global setting change detected");
+                                            match n.payload() {
+                                                BASE_URL_SETTING => {
+                                                    tracing::info!("Base URL setting change detected");
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload base url setting");
+                                                    }
+                                                },
+                                                OAUTH_SETTING => {
+                                                    tracing::info!("OAuth setting change detected");
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload oauth setting");
+                                                    }
+                                                },
+                                                CUSTOM_TAGS_SETTING => {
+                                                    tracing::info!("Custom tags setting change detected");
+                                                    if let Err(e) = reload_custom_tags_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload custom tags setting");
+                                                    }
+                                                },
+                                                LICENSE_KEY_SETTING => {
+                                                    tracing::info!("License Key setting change detected");
+                                                    if let Err(e) = reload_license_key(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload license key setting");
+                                                    }
+                                                },
+                                                RETENTION_PERIOD_SECS_SETTING => {
+                                                    tracing::info!("Retention period setting change detected");
+                                                    reload_retention_period_setting(&db).await
+                                                },
+                                                REQUEST_SIZE_LIMIT_SETTING => {
+                                                    tracing::info!("Request limit size change detected, killing server expecting to be restarted");
+                                                    // we wait a bit randomly to avoid having all servers shutdown at same time
+                                                    let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                    if let Err(e) = tx.send(()) {
+                                                        tracing::error!(error = %e, "Could not send killpill to server");
+                                                    }
+                                                }
+                                                a @_ => {
+                                                    tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
+                                                }
+                                            }
+                                        },
+                                        _ => {
+                                            tracing::warn!("Unknown notification received");
+                                            continue;
+                                        }
+                                    }
+                                },
+                                Err(e) => {
+                                    tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
+                                    listener = retry_listen_pg(&db).await;
+                                    continue;
+                                }
+                            };
+                        },
+                        _ = rx.recv() => {
+                                println!("received killpill for monitor job");
+                                break;
+                        }
+                    }
+                }
+            });
+
             Ok(()) as anyhow::Result<()>
         };
 
         let metrics_f = async {
-            match metrics_addr {
-                Some(_addr) => {
-                    #[cfg(not(feature = "enterprise"))]
-                    panic!("Metrics are only available in the Enterprise Edition");
+            if let Some(_addr) = metrics_addr {
+                #[cfg(not(feature = "enterprise"))]
+                panic!("Metrics are only available in the Enterprise Edition");
 
-                    #[cfg(feature = "enterprise")]
-                    windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
-                        .await
-                        .map_err(anyhow::Error::from)
-                }
-                None => Ok(()),
+                #[cfg(feature = "enterprise")]
+                windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
             }
+            Ok(()) as anyhow::Result<()>
         };
 
         futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
@@ -208,6 +322,40 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
+async fn listen_pg(db: &DB) -> Option<PgListener> {
+    let mut listener = match PgListener::connect_with(&db).await {
+        Ok(l) => l,
+        Err(e) => {
+            tracing::error!(error = %e, "Could not connect to database");
+            return None;
+        }
+    };
+
+    if let Err(e) = listener
+        .listen_all(vec!["notify_config_change", "notify_global_setting_change"])
+        .await
+    {
+        tracing::error!(error = %e, "Could not listen to database");
+        return None;
+    }
+
+    return Some(listener);
+}
+
+async fn retry_listen_pg(db: &DB) -> PgListener {
+    let mut listener = listen_pg(db).await;
+    loop {
+        if listener.is_none() {
+            tracing::info!("Retrying listening to pg listen in 5 seconds");
+            tokio::time::sleep(Duration::from_secs(5)).await;
+            listener = listen_pg(db).await;
+        } else {
+            tracing::info!("Successfully connected to pg listen");
+            return listener.unwrap();
+        }
+    }
+}
+
 fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
@@ -225,40 +373,14 @@ fn display_config(envs: &[&str]) {
     )
 }
 
-pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
-    db: &Pool<Postgres>,
-    rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
-) -> tokio::task::JoinHandle<()> {
-    let db1 = db.clone();
-    let db2 = db.clone();
-
-    let rx2 = rx.resubscribe();
-    let base_internal_url = base_internal_url.to_string();
-    tokio::spawn(async move {
-        join!(
-            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
-            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
-        );
-    })
-}
-
 pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
+    tx: tokio::sync::broadcast::Sender<()>,
     num_workers: i32,
     base_internal_url: String,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
-    #[cfg(feature = "enterprise")]
-    ee::verify_license_key(LICENSE_KEY.clone())?;
-
-    #[cfg(not(feature = "enterprise"))]
-    if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) {
-        panic!("License key is required ONLY for the enterprise edition");
-    }
-
     let instance_name = gethostname()
         .to_str()
         .map(|x| {
@@ -320,6 +442,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
         let ip = ip.clone();
         let rx = rx.resubscribe();
+        let tx = tx.clone();
         let base_internal_url = base_internal_url.clone();
         let rsmq2 = rsmq.clone();
         let sync_barrier = sync_barrier.clone();
@@ -333,6 +456,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 num_workers as u32,
                 &ip,
                 rx,
+                tx,
                 &base_internal_url,
                 rsmq2,
                 sync_barrier,

backend/src/monitor.rs

@@ -1,18 +1,41 @@
-use std::time::Duration;
+use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc};
 
 use once_cell::sync::OnceCell;
+use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
-use tokio::sync::mpsc;
+use tokio::{
+    join,
+    sync::{mpsc, RwLock},
+};
 use uuid::Uuid;
+use windmill_api::{
+    oauth2::{build_oauth_clients, OAuthClient},
+    DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT,
+};
 use windmill_common::{
     error,
+    global_settings::{
+        BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        RETENTION_PERIOD_SECS_SETTING,
+    },
     jobs::{JobKind, QueuedJob},
-    METRICS_ENABLED,
+    server::load_server_config,
+    users::truncate_token,
+    worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
+    BASE_URL, DB, METRICS_ENABLED,
 };
 use windmill_worker::{
     create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
 };
 
+#[cfg(feature = "enterprise")]
+use crate::ee::verify_license_key;
+
+#[cfg(feature = "enterprise")]
+use windmill_api::LICENSE_KEY_VALID;
+
+use crate::ee::set_license_key;
+
 lazy_static::lazy_static! {
     static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
     .ok()
@@ -35,27 +58,417 @@ lazy_static::lazy_static! {
         "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
     )
     .unwrap();
+
+    static ref QUEUE_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
+        "queue_count",
+        "Number of jobs in the queue",
+        &["tag"]
+    ).unwrap();
+
+    static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
+
 }
 
-pub async fn handle_zombie_jobs_periodically<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
+pub async fn initial_load(
     db: &Pool<Postgres>,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
+    tx: tokio::sync::broadcast::Sender<()>,
+    worker_mode: bool,
+    server_mode: bool,
 ) {
-    loop {
-        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
-            _ = rx.recv() => {
-                    println!("received killpill for monitor job");
-                    break;
+    let reload_worker_config_f = async {
+        if worker_mode {
+            reload_worker_config(&db, tx, false).await;
+        }
+    };
+    let reload_custom_tags_f = async {
+        if server_mode {
+            if let Err(e) = reload_custom_tags_setting(db).await {
+                tracing::error!("Error reloading custom tags: {:?}", e)
             }
         }
+    };
+
+    let reload_base_url_f = async {
+        
+            if let Err(e) = reload_base_url_setting(db).await {
+                tracing::error!("Error reloading base url: {:?}", e)
+            }
+        
+    };
+
+    let reload_server_config_f = async {
+        if server_mode {
+            reload_server_config(&db).await;
+        }
+    };
+    let reload_retention_period_f = async {
+        if server_mode {
+            reload_retention_period_setting(&db).await;
+        }
+    };
+
+    let reload_request_size_f = async {
+        if server_mode {
+            reload_request_size(&db).await;
+        }
+    };
+
+    let reload_license_key_f = async {
+        if server_mode {
+            #[cfg(feature = "enterprise")]
+            if let Err(e) = reload_license_key(&db).await {
+                tracing::error!("Error reloading license key: {:?}", e)
+            }
+        }
+    };
+
+    join!(
+        reload_worker_config_f,
+        reload_server_config_f,
+        reload_custom_tags_f,
+        reload_request_size_f,
+        reload_base_url_f,
+        reload_retention_period_f,
+        reload_license_key_f
+    );
+}
+
+pub async fn delete_expired_items(db: &DB) -> () {
+    let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
+        "DELETE FROM token WHERE expiration <= now()
+        RETURNING concat(substring(token for 10), '*****')",
+    )
+    .fetch_all(db)
+    .await;
+
+    match tokens_deleted_r {
+        Ok(tokens) => {
+            if tokens.len() > 0 {
+                tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
     }
+
+    let pip_resolution_r = sqlx::query_scalar!(
+        "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
+    )
+    .fetch_all(db)
+    .await;
+
+    match pip_resolution_r {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
+    }
+
+    let deleted_cache = sqlx::query_scalar!(
+            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
+        )
+        .fetch_all(db)
+        .await;
+
+    match deleted_cache {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} cache resource: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
+    }
+
+    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
+    if job_retention_secs > 0 {
+        let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                job_retention_secs
+            )
+            .fetch_all(db)
+            .await;
+
+        match deleted_jobs {
+            Ok(deleted_jobs) => {
+                if deleted_jobs.len() > 0 {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+                        deleted_jobs.len(),
+                        job_retention_secs,
+                        deleted_jobs,
+                    )
+                }
+            }
+            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+        }
+    }
+}
+
+pub async fn reload_retention_period_setting(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        RETENTION_PERIOD_SECS_SETTING,
+        "JOB_RETENTION_SECS",
+        60 * 60 * 24 * 60,
+        JOB_RETENTION_SECS.clone(),
+        |x| x,
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_request_size(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        REQUEST_SIZE_LIMIT_SETTING,
+        "REQUEST_SIZE_LIMIT",
+        DEFAULT_BODY_LIMIT,
+        REQUEST_SIZE_LIMIT.clone(),
+        |x| x.mul(1024 * 1024),
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_license_key(db: &DB) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        LICENSE_KEY_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var("LICENSE_KEY")
+        .ok()
+        .and_then(|x| x.parse::<String>().ok())
+        .unwrap_or(String::new());
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting LICENSE_KEY from db config: {}",
+                truncate_token(&v)
+            );
+            value = v;
+        } else {
+            tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value);
+        }
+    };
+
+    set_license_key(value).await?;
+
+    Ok(())
+}
+
+pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    default: T,
+    lock: Arc<RwLock<T>>,
+    transformer: fn(T) -> T,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var)
+        .ok()
+        .and_then(|x| x.parse::<T>().ok())
+        .unwrap_or(default);
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<T>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = transformer(v);
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+    server_mode: bool,
+) {
+    let zombie_jobs_f = async {
+        if server_mode {
+            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+        }
+    };
+    let expired_items_f = async {
+        if server_mode {
+            delete_expired_items(&db).await;
+        }
+    };
+
+    let verify_license_key_f = async {
+        if server_mode {
+            #[cfg(feature = "enterprise")]
+            if let Err(e) = verify_license_key().await {
+                tracing::error!("Error verifying license key: {:?}", e);
+                let mut l = LICENSE_KEY_VALID.write().await;
+                *l = false;
+            }
+        }
+    };
+
+    let expose_queue_metrics_f = async {
+        if *METRICS_ENABLED && server_mode {
+            expose_queue_metrics(&db).await;
+        }
+    };
+    join!(
+        expired_items_f,
+        zombie_jobs_f,
+        expose_queue_metrics_f,
+        verify_license_key_f
+    );
+}
+
+pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
+    let queue_counts = sqlx::query!(
+        "SELECT tag, count(*) as count FROM queue WHERE
+        scheduled_for <= now() - ('3 seconds')::interval AND running = false
+        GROUP BY tag"
+    )
+    .fetch_all(db)
+    .await
+    .ok()
+    .unwrap_or_else(|| vec![]);
+    for q in queue_counts {
+        let count = q.count.unwrap_or(0);
+        let tag = q.tag;
+        let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+        metric.set(count as i64);
+    }
+}
+
+pub async fn reload_server_config(db: &Pool<Postgres>) {
+    let config = load_server_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading server config: {:?}", e)
+    } else {
+        let mut wc = SERVER_CONFIG.write().await;
+        tracing::info!("Reloading server config...");
+        *wc = config.unwrap()
+    }
+}
+
+pub async fn reload_worker_config(
+    db: &DB,
+    tx: tokio::sync::broadcast::Sender<()>,
+    kill_if_change: bool,
+) {
+    let config = load_worker_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading worker config: {:?}", e)
+    } else {
+        let wc = WORKER_CONFIG.read().await;
+        let config = config.unwrap();
+        if *wc != config {
+            if kill_if_change && (*wc).dedicated_worker != config.dedicated_worker {
+                tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
+                let _ = tx.send(());
+            }
+            drop(wc);
+
+            let mut wc = WORKER_CONFIG.write().await;
+            tracing::info!("Reloading worker config...");
+            *wc = config
+        }
+    }
+}
+
+pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
+    let q_base_url = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        BASE_URL_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let base_url = if let Some(q) = q_base_url {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse base_url setting as a string, found: {:#?}",
+                &q.value
+            );
+            std::env::var("BASE_URL")
+                .ok()
+                .unwrap_or_else(|| "http://localhost".to_string())
+        }
+    } else {
+        std::env::var("BASE_URL")
+            .ok()
+            .unwrap_or_else(|| "http://localhost".to_string())
+    };
+
+    let q_oauth = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        OAUTH_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let oauths = if let Some(q) = q_oauth {
+        if let Ok(v) =
+            serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.value.clone())
+        {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse oauth setting as a json, found: {:#?}",
+                &q.value
+            );
+            None
+        }
+    } else {
+        None
+    };
+
+    let is_secure = base_url.starts_with("https://");
+
+    {
+        let mut l = OAUTH_CLIENTS.write().await;
+        *l = build_oauth_clients(&base_url, oauths)
+        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
+        .unwrap();
+    }
+
+    {
+        let mut l = BASE_URL.write().await;
+        *l = base_url
+    }
+
+    {
+        let mut l = IS_SECURE.write().await;
+        *l = is_secure;
+    }
+
+    Ok(())
 }
 
 async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(

backend/tests/fixtures/base.sql

@@ -17,59 +17,6 @@ INSERT INTO workspace_settings (workspace_id) VALUES
 
 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main(fail: boolean = true) {
-  if (fail) {
-    throw new Error("Failed")
-  }
-
-  return "OK"
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_script', -28028598712388162, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Error handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
-'',
-'',
-'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Recovery handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
-'',
-'',
-'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
-
-INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
-'test-workspace',
-'system',
-'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_flow'
-);
-
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 

backend/tests/fixtures/relative_bun.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'bun', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'bun', '');

backend/tests/fixtures/relative_deno.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'deno', '');

backend/tests/fixtures/relative_python.sql

@@ -0,0 +1,41 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system/same_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system_relative/different_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+from f.system.same_folder_script import main as test1
+from ..system.same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from .different_folder_script import main as test4
+
+def main():
+    return [test1(), test2(), test3(), test4()]
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'python3', '');

backend/tests/fixtures/schedule.sql

@@ -0,0 +1,52 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);

backend/tests/worker.rs

@@ -1,26 +1,81 @@
 use std::sync::Arc;
 
+#[cfg(feature = "enterprise")]
 use chrono::Timelike;
 use futures::StreamExt;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
-use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::{
-    sync::RwLock,
-    time::{timeout, Duration},
-};
-use windmill_api::jobs::{CompletedJob, Job};
+use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
+use tokio::sync::RwLock;
+
+#[cfg(feature = "enterprise")]
+use tokio::time::{timeout, Duration};
+
 use windmill_api_client::types::{
-    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+    CreateFlowBody, RawScript
 };
+
+use sqlx::query;
+
+#[cfg(feature = "enterprise")]
+use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
+
+use windmill_api_client::types::{NewScript, NewScriptLanguage};
+
+use windmill_common::worker::WORKER_CONFIG;
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
-    jobs::{JobPayload, RawCode},
-    scripts::ScriptLang,
+    jobs::{JobPayload, RawCode, JobKind},
+    scripts::{ScriptLang, ScriptHash}
 };
-use windmill_queue::{get_queued_job, PushIsolationLevel};
+use windmill_queue::PushIsolationLevel;
+use serde::Serialize;
+
+use std::str::FromStr;
+
+#[derive(Debug, sqlx::FromRow, Serialize)]
+pub struct CompletedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub started_at: chrono::DateTime<chrono::Utc>,
+    pub duration_ms: i64,
+    pub success: bool,
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    pub result: Option<serde_json::Value>,
+    pub logs: Option<String>,
+    pub deleted: bool,
+    pub raw_code: Option<String>,
+    pub canceled: bool,
+    pub canceled_by: Option<String>,
+    pub canceled_reason: Option<String>,
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    pub flow_status: Option<serde_json::Value>,
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    pub is_skipped: bool,
+    pub email: String,
+    pub visible_to_owner: bool,
+    pub mem_peak: Option<i32>,
+    pub tag: String,
+    pub script_hash: Option<ScriptHash>,
+    pub language: Option<ScriptLang>,
+    pub job_kind: JobKind,
+
+}
+
+impl CompletedJob {
+    pub fn json_result(&self) -> Option<serde_json::Value> {
+        self.result.clone()
+    }
+}
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -54,37 +109,6 @@ fn next_worker_name() -> String {
     format!("{id}/{thread_name}")
 }
 
-pub async fn get_job_by_id<'c>(
-    mut tx: Transaction<'c, Postgres>,
-    w_id: &str,
-    id: Uuid,
-) -> windmill_common::error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
-    let cjob_option = sqlx::query_as::<_, CompletedJob>(
-        "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-    )
-    .bind(id)
-    .bind(w_id)
-    .fetch_optional(&mut *tx)
-    .await?;
-    let job_option = match cjob_option {
-        Some(job) => Some(Job::CompletedJob(job)),
-        None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
-    };
-    if job_option.is_some() {
-        Ok((job_option, tx))
-    } else {
-        // check if a job had been moved in-between queries
-        let cjob_option = sqlx::query_as::<_, CompletedJob>(
-            "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-        )
-        .bind(id)
-        .bind(w_id)
-        .fetch_optional(&mut *tx)
-        .await?;
-        Ok((cjob_option.map(Job::CompletedJob), tx))
-    }
-}
-
 pub struct ApiServer {
     pub addr: std::net::SocketAddr,
     tx: tokio::sync::broadcast::Sender<()>,
@@ -115,21 +139,22 @@ impl ApiServer {
     }
 
     async fn close(self) -> anyhow::Result<()> {
+        println!("closing api server");
         let Self { tx, task, .. } = self;
         drop(tx);
         task.await.unwrap()
     }
 }
 
-async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
-    tracing::info!(
-        "{:#?}",
-        get_job_by_id(db.begin().await?, "test-workspace", id)
-            .await?
-            .0
-    );
-    Ok(())
-}
+// async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
+//     tracing::info!(
+//         "{:#?}",
+//         get_job_by_id(db.begin().await?, "test-workspace", id)
+//             .await?
+//             .0
+//     );
+//     Ok(())
+// }
 
 fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     cjob.flow_status.clone().and_then(|fs| {
@@ -299,7 +324,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!({
@@ -337,7 +362,7 @@ mod suspend_resume {
             .arg("port", json!(port))
             .run_until_complete(&db, port)
             .await
-            .result
+            .json_result()
             .unwrap();
 
         server.close().await.unwrap();
@@ -400,7 +425,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
@@ -555,7 +580,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -584,7 +609,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -626,7 +651,7 @@ def main(last, port):
             .run_until_complete(&db, server.addr.port())
             .await;
 
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(server.close().await, attempts);
         assert!(result["error"]
             .as_object()
@@ -690,7 +715,7 @@ def main(error, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await;
-        let result = cjob.result.clone().unwrap();
+        let result = cjob.json_result().clone().unwrap();
         let failed_module = get_module(&cjob, "a").unwrap();
         match failed_module {
             FlowStatusModule::Failure { .. } => {}
@@ -748,7 +773,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -757,7 +782,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!((0..257).collect::<Vec<_>>()))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
@@ -805,7 +830,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -815,7 +840,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .run_until_complete(&db, server.addr.port())
         .await;
     // println!("{:#?}", job);
-    let result = job.result.unwrap();
+    let result = job.json_result().unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
         .as_object()
@@ -855,7 +880,7 @@ impl RunJob {
             args,
             /* user */ "test-user",
             /* email  */ "test@windmill.dev",
-            /* permissioned_as */ "u/admin".to_string(),
+            /* permissioned_as */ "u/test-user".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
@@ -881,7 +906,8 @@ impl RunJob {
         let uuid = self.push(db).await;
         let listener = listen_for_completed_jobs(db).await;
         in_test_worker(db, listener.find(&uuid), port).await;
-        completed_job(uuid, db).await
+        let r = completed_job(uuid, db).await;
+        r
     }
 }
 
@@ -916,13 +942,12 @@ async fn in_test_worker<Fut: std::future::Future>(
     };
 
     /* ensure the worker quits before we return */
-    drop(quit);
+    quit.send(()).expect("send");
 
     let _: () = worker
         .await
         .expect("worker timed out")
         .expect("worker panicked");
-
     res
 }
 
@@ -946,8 +971,13 @@ fn spawn_test_worker(
     let worker_name: String = next_worker_name();
     let ip: &str = Default::default();
 
+    let tx2 = tx.clone();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
+        {
+        let mut wc = WORKER_CONFIG.write().await;
+        (*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
+        }
         windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
             &db,
             worker_instance,
@@ -956,6 +986,7 @@ fn spawn_test_worker(
             1,
             ip,
             rx,
+            tx2,
             &base_internal_url,
             None,
             Arc::new(RwLock::new(None)),
@@ -995,8 +1026,8 @@ async fn listen_for_uuid_on(
 }
 
 async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
-    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-        .bind(uuid)
+
+    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job  WHERE id = $1").bind(uuid)
         .fetch_one(db)
         .await
         .unwrap()
@@ -1106,7 +1137,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
         println!("deno flow iteration: {}", i);
         let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await;
         // println!("job: {:#?}", job.flow_status);
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i);
     }
 }
@@ -1142,7 +1173,7 @@ async fn test_identity(db: Pool<Postgres>) {
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!(42));
 }
@@ -1331,7 +1362,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
 
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(
         result,
@@ -1386,7 +1417,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
     let job = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([[42]]));
 }
@@ -1431,7 +1462,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw 123"), result);
 
@@ -1440,7 +1471,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!(-123), result);
 }
 
@@ -1489,7 +1520,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw [123]"), result);
 
@@ -1498,7 +1529,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!([-123]), result);
 }
 
@@ -1552,7 +1583,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}");
@@ -1587,7 +1618,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}");
@@ -1624,7 +1655,7 @@ func main(derp string) (string, error) {
     .arg("derp", json!("world"))
     .run_until_complete(&db, port)
     .await
-    .result
+    .json_result()
     .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1654,8 +1685,7 @@ echo "hello $msg"
     .arg("msg", json!("world"))
     .run_until_complete(&db, port)
     .await;
-
-    assert_eq!(job.result, Some(json!("hello world")));
+    assert_eq!(job.json_result(), Some(json!("hello world")));
 }
 
 #[sqlx::test(fixtures("base"))]
@@ -1682,7 +1712,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1715,7 +1745,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(3));
@@ -1747,7 +1777,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("test-workspace"));
@@ -1803,7 +1833,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(0));
@@ -1843,7 +1873,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
     let job = run_job_in_new_worker_until_complete(&db, flow, port).await;
 
     assert_eq!(
-        job.result.unwrap(),
+        job.json_result().unwrap(),
         serde_json::json!( {"error":  {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
     );
 }
@@ -1884,7 +1914,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([]));
@@ -1939,7 +1969,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(9));
@@ -2007,7 +2037,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2]));
@@ -2043,7 +2073,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 3]));
@@ -2081,7 +2111,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2118,7 +2148,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2165,7 +2195,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2202,7 +2232,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2266,7 +2296,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2, 3]));
@@ -2323,7 +2353,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     println!("{:#?}", result);
@@ -2388,7 +2418,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(0))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2404,7 +2434,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(1))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2420,7 +2450,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(2))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2436,7 +2466,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(3))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
@@ -2586,7 +2616,9 @@ async fn test_rust_client(db: Pool<Postgres>) {
     .unwrap();
 }
 
-#[sqlx::test(fixtures("base"))]
+
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_script_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2642,8 +2674,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2702,8 +2733,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2721,7 +2751,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
 }
 
 
-#[sqlx::test(fixtures("base"))]
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2778,8 +2809,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2839,8 +2869,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2856,3 +2885,203 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     )
     .await;
 }
+
+
+async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    client.create_script(
+        "test-workspace",
+        &NewScript {
+            language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
+            content: script_content,
+            path: "f/system/test_import".to_string(),
+            concurrent_limit: vec![],
+            concurrency_time_window_s: vec![],
+            cache_ttl: None,
+            dedicated_worker: None,
+            description: "".to_string(),
+            draft_only: None,
+            envs: vec![],
+            is_template: None,
+            kind: None,
+            parent_hash: None,
+            lock: vec![],
+            summary: "".to_string(),
+            tag: None,
+            schema: std::collections::HashMap::new(),
+        },
+    ).await.unwrap();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        completed.next().await; // deployed script
+
+        let script =
+            query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string())
+                .fetch_one(&db2)
+                .await
+                .unwrap();
+
+        let job = RunJob::from(JobPayload::ScriptHash {
+            path: "f/system/test_import".to_string(),
+            hash: ScriptHash(script.hash),
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+            dedicated_worker: None,
+            language,
+        }).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        let job = RunJob::from(JobPayload::Code(RawCode {
+            content: script_content,
+            path: Some("f/system/test_import".to_string()),
+            language,
+            lock: None,
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+        })).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_relative_imports_bun(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_nested_imports_bun(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_relative_imports_deno(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_nested_imports_deno(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_relative_imports_python(db: Pool<Postgres>) {
+    let content = r#"
+from f.system.same_folder_script import main as test1
+from .same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from ..system_relative.different_folder_script import main as test4
+    
+def main():
+    return [test1(), test2(), test3(), test4()]
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_nested_imports_python(db: Pool<Postgres>) {
+
+    let content = r#"
+
+from f.system_relative.nested_script import main as test
+
+def main():
+    return test()
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+}

backend/windmill-api/Cargo.toml

@@ -53,7 +53,7 @@ cron.workspace = true
 mime_guess.workspace = true
 rust-embed.workspace = true
 tracing-subscriber.workspace = true
-retainer.workspace = true
+quick_cache.workspace = true
 rand.workspace = true
 time.workspace = true
 magic-crypt.workspace = true
@@ -74,3 +74,4 @@ bytes.workspace = true
 mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
+rsa.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.168.3
+  version: 1.177.0
   title: Windmill API
 
   contact:
@@ -575,6 +575,81 @@ paths:
             application/json:
               schema: {}
 
+  /settings/test_smtp:
+    post:
+      summary: test smtp
+      operationId: testSmtp
+      tags:
+        - setting
+      requestBody:
+        description: test smtp payload
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                to:
+                  type: string
+                smtp:
+                  type: object
+                  properties:
+                    host:
+                      type: string
+                    username:
+                      type: string
+                    password:
+                      type: string
+                    port:
+                      type: integer
+                    from:
+                      type: string
+                    tls_implicit:
+                      type: boolean
+                  required:
+                    - host
+                    - username
+                    - password
+                    - port
+                    - from
+                    - tls_implicit
+              required:
+                - to
+                - smtp
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
+  /settings/test_license_key:
+    post:
+      summary: test license key
+      operationId: testLicenseKey
+      tags:
+        - setting
+      requestBody:
+        description: test license key
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                license_key:
+                  type: string
+              required:
+                - license_key
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -2386,6 +2461,46 @@ paths:
                   - content
                   - language
 
+  /scripts/hub/query:
+    get:
+      summary: query hub scripts by similarity
+      operationId: queryHubScripts
+      tags:
+        - script
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: kind
+          description: query scripts kind
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
   /w/{workspace}/scripts/list:
     get:
       summary: list all available scripts
@@ -3479,6 +3594,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
+  /w/{workspace}/apps_u/public_resource/{path}:
+    get:
+      summary: get public resource
+      operationId: get public resource
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -3914,6 +4045,27 @@ paths:
                 required:
                   - database_length
 
+  /w/{workspace}/jobs/completed/count:
+    get:
+      summary: get completed count
+      operationId: getCompletedCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: completed count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
   /w/{workspace}/jobs/queue/cancel_all:
     post:
       summary: cancel all jobs
@@ -3991,6 +4143,7 @@ paths:
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/CreatedOrStartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
@@ -5191,6 +5344,80 @@ paths:
                 items:
                   $ref: "#/components/schemas/WorkerPing"
 
+  /configs/list_worker_groups:
+    get:
+      summary: list worker groups
+      operationId: listWorkerGroups
+      tags:
+        - config
+      responses:
+        "200":
+          description: a list of worker group configs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    config: {}
+                  required:
+                    - name
+                    - config
+
+  /configs/get/{name}:
+    get:
+      summary: get config
+      operationId: get config
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: a config
+          content:
+            application/json:
+              schema: {}
+
+  /configs/update/{name}:
+    post:
+      summary: Update config
+      operationId: updateConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      requestBody:
+        description: worker group
+        required: true
+        content:
+          application/json:
+            schema: {}
+      responses:
+        "200":
+          description: Update a worker group
+          content:
+            text/plain:
+              schema:
+                type: string
+    delete:
+      summary: Delete Config
+      operationId: deleteConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: Delete config
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/acls/get/{kind}/{path}:
     get:
       summary: get granular acls
@@ -7111,6 +7338,8 @@ components:
           type: array
           items:
             type: string
+        worker_group:
+          type: string
       required:
         - worker
         - worker_instance
@@ -7118,6 +7347,7 @@ components:
         - started_at
         - ip
         - jobs_executed
+        - worker_group
 
     UserWorkspaceList:
       type: object

backend/windmill-api/src/apps.rs

@@ -57,6 +57,7 @@ pub fn unauthed_service() -> Router {
     Router::new()
         .route("/execute_component/*path", post(execute_component))
         .route("/public_app/:secret", get(get_public_app_by_secret))
+        .route("/public_resource/*path", get(get_public_resource))
 }
 
 pub fn global_service() -> Router {
@@ -341,6 +342,27 @@ async fn get_public_app_by_secret(
     Ok(Json(app))
 }
 
+async fn get_public_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    if !path.starts_with("f/app_themes/") {
+        return Err(Error::BadRequest(
+            "Only app themes are public resources".to_string(),
+        ));
+    }
+    let res = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten();
+    Ok(Json(res))
+}
+
 async fn get_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -508,6 +530,7 @@ pub async fn get_hub_app_by_id(
         &format!("https://hub.windmill.dev/apps/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()

backend/windmill-api/src/capture.rs

@@ -9,7 +9,7 @@
 use axum::{
     extract::{Extension, Path, Query},
     routing::{get, post, put},
-    Json, Router,
+    Router,
 };
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
@@ -21,7 +21,7 @@ use windmill_common::{
 
 use crate::{
     db::{ApiAuthed, DB},
-    jobs::add_include_headers,
+    jobs::{add_include_headers, add_raw_string, JsonOrForm},
 };
 
 const KEEP_LAST: i64 = 8;
@@ -95,11 +95,12 @@ pub async fn update_payload(
     Path((w_id, path)): Path<(String, StripPath)>,
     Query(run_query): Query<IncludeHeaderQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
-
     let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
+
     sqlx::query!(
         "
        UPDATE capture

backend/windmill-api/src/configs.rs

@@ -0,0 +1,114 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use axum::{
+    extract::{Extension, Path},
+    routing::{get, post},
+    Json, Router,
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use windmill_common::{
+    db::UserDB,
+    error::{self},
+    DB,
+};
+
+use crate::{db::ApiAuthed, utils::require_super_admin};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/list_worker_groups", get(list_worker_groups))
+        .route("/update/:name", post(update_config).delete(delete_config))
+        .route("/get/:name", get(get_config))
+}
+
+#[derive(Serialize, Deserialize, FromRow)]
+struct Config {
+    name: String,
+    config: serde_json::Value,
+}
+
+async fn list_worker_groups(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Vec<Config>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+        .fetch_all(&db)
+        .await?;
+    Ok(Json(rows))
+}
+
+async fn get_config(
+    authed: ApiAuthed,
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Option<serde_json::Value>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
+        .fetch_optional(&db)
+        .await?
+        .map(|c| c.config);
+
+    Ok(Json(config))
+}
+
+async fn update_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(config): Json<serde_json::Value>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    #[cfg(not(feature = "enterprise"))]
+    if name.starts_with("worker__") {
+        return Err(error::Error::BadRequest(
+            "Worker groups configurable from UI available only in the enterprise version"
+                .to_string(),
+        ));
+    }
+
+    sqlx::query!(
+        "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+        &name,
+        config
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Updated config {name}"))
+}
+
+async fn delete_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    let tx = user_db.begin(&authed).await?;
+
+    require_super_admin(&db, &authed.email).await?;
+    tx.commit().await?;
+
+    let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name)
+        .fetch_all(&db)
+        .await?;
+
+    if deleted.len() == 0 {
+        return Err(error::Error::NotFound(format!(
+            "Config {name} not found",
+            name = name
+        )));
+    }
+    Ok(format!("Deleted config {name}"))
+}

backend/windmill-api/src/ee.rs

@@ -0,0 +1,41 @@
+use anyhow::anyhow;
+use base64::Engine;
+use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
+use sha2::Sha256;
+
+pub async fn validate_license_key(license_key: String) -> anyhow::Result<String> {
+    let mut splitted_lk = license_key.split(".");
+    if splitted_lk.clone().count() != 3 {
+        return Err(anyhow!(
+            "license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)"
+        ));
+    }
+
+    let id = splitted_lk.next().unwrap();
+    let expiry = splitted_lk.next().unwrap();
+    let signature_b64 = splitted_lk.next().unwrap();
+
+    const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
+    let pub_key = rsa::RsaPublicKey::from_public_key_der(
+        &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
+    )?;
+    let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
+    rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
+        .verify(
+            &format!("{id}{expiry}").as_bytes(),
+            &rsa::pss::Signature::from(signature),
+        )
+        .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
+
+    let expiry_nb = expiry.parse::<u64>()?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        return Err(anyhow!("License key expired".to_string()));
+    };
+
+    Ok(id.to_string())
+}

backend/windmill-api/src/flows.rs

@@ -160,6 +160,7 @@ pub async fn get_hub_flow_by_id(
         &format!("https://hub.windmill.dev/flows/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()

backend/windmill-api/src/groups.rs

@@ -15,6 +15,7 @@ use axum::{
     Json, Router,
 };
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::worker::CLOUD_HOSTED;
 use windmill_common::{db::UserDB, users::username_to_permissioned_as};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -23,7 +24,6 @@ use windmill_common::{
 
 use serde::{Deserialize, Serialize};
 use sqlx::{query_scalar, FromRow, Postgres, Transaction};
-use windmill_queue::CLOUD_HOSTED;
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -212,6 +212,7 @@ async fn _check_nb_of_groups(db: &DB) -> Result<()> {
     }
     return Ok(());
 }
+
 async fn create_group(
     authed: ApiAuthed,
     Extension(_db): Extension<DB>,

backend/windmill-api/src/inputs.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{db::ApiAuthed, jobs::CompletedJob};
+use crate::db::ApiAuthed;
 use axum::{
     extract::{Path, Query},
     routing::{get, post},
@@ -15,7 +15,7 @@ use axum::{
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use sqlx::types::Uuid;
+use sqlx::{types::Uuid, FromRow};
 use std::{
     fmt::{Display, Formatter},
     vec,
@@ -102,6 +102,15 @@ pub struct Input {
     success: bool,
 }
 
+#[derive(Debug, Serialize, Deserialize, FromRow)]
+pub struct CompletedJobMini {
+    id: Uuid,
+    created_at: chrono::DateTime<chrono::Utc>,
+    args: Option<serde_json::Value>,
+    created_by: String,
+    success: bool,
+}
+
 async fn get_input_history(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -114,13 +123,13 @@ async fn get_input_history(
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = &format!(
-        "select * from completed_job \
+        "select id, created_at, created_by, args, success from completed_job \
         where {} = $1 and job_kind = $2 and workspace_id = $3 \
         order by created_at desc limit $4 offset $5",
         r.runnable_type.column_name()
     );
 
-    let query = sqlx::query_as::<_, CompletedJob>(sql);
+    let query = sqlx::query_as::<_, CompletedJobMini>(sql);
 
     let query = match r.runnable_type {
         RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),

backend/windmill-api/src/lib.rs

@@ -11,9 +11,8 @@ use crate::oauth2::AllClients;
 use crate::saml::{SamlSsoLogin, ServiceProviderExt};
 use crate::scim::has_scim_token;
 use crate::tracing_init::MyOnFailure;
-use crate::workers::ALL_TAGS;
 use crate::{
-    oauth2::{build_oauth_clients, SlackVerifier},
+    oauth2::SlackVerifier,
     tracing_init::{MyMakeSpan, MyOnResponse},
     users::OptAuthed,
     webhook_util::WebhookShared,
@@ -25,9 +24,10 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
 use hyper::{http, Method};
-use mail_send::SmtpClientBuilder;
 use reqwest::Client;
+use std::collections::HashMap;
 use std::{net::SocketAddr, sync::Arc};
+use tokio::sync::RwLock;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
 use tower_http::{
@@ -36,14 +36,18 @@ use tower_http::{
 };
 use windmill_common::db::UserDB;
 use windmill_common::utils::rd_string;
+use windmill_common::worker::ALL_TAGS;
+use windmill_common::BASE_URL;
 
 use windmill_common::error::AppError;
 
 mod apps;
 mod audit;
 mod capture;
+mod configs;
 mod db;
 mod drafts;
+pub mod ee;
 mod favorite;
 mod flows;
 mod folders;
@@ -51,7 +55,7 @@ mod granular_acls;
 mod groups;
 mod inputs;
 pub mod jobs;
-mod oauth2;
+pub mod oauth2;
 mod openai;
 mod raw_apps;
 mod resources;
@@ -72,16 +76,11 @@ mod workspaces;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-pub use users::delete_expired_items_perdiodically;
+pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
 
-pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
 lazy_static::lazy_static! {
-    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
-    pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
-    .ok()
-    .and_then(|x| x.parse::<usize>().ok())
-    .unwrap_or(DEFAULT_BODY_LIMIT);
+    pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
 
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
@@ -90,59 +89,22 @@ lazy_static::lazy_static! {
         .ok()
         .map(|x| SlackVerifier::new(x).unwrap());
 
-        static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
+    pub static ref IS_SECURE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
 
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
         .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
         .build().unwrap();
 
-    pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
-        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
-        .unwrap();
+    pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
+        logins: HashMap::new(),
+        connects: HashMap::new(),
+        slack: None
+    }));
 
-    pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
-        let smtp = parse_smtp();
-        if let Some(smtp) = smtp {
-            match smtp {
-                Ok(smtp) => Some(smtp),
-                Err(e) => {
-                    tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e);
-                    None
-                }
-            }
-        } else {
-            tracing::warn!("SMTP is not configured, emails will not be sent");
-            None
-        }
-    };
-
-    pub static ref SMTP_FROM: String = std::env::var("SMTP_FROM").unwrap_or_else(|_| "noreply@getwindmill.com".to_string());
-
-    pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
-}
-
-pub fn parse_smtp() -> Option<windmill_common::error::Result<SmtpClientBuilder<String>>> {
-    let username = std::env::var("SMTP_USERNAME").ok();
-    let port = std::env::var("SMTP_PORT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(587);
-    let password = std::env::var("SMTP_PASSWORD").ok();
-    let host = std::env::var("SMTP_HOST").ok();
-    let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(false);
-
-    if username.is_some() && password.is_some() && host.is_some() {
-        let smtp = SmtpClientBuilder::new(host.unwrap(), port)
-            .implicit_tls(tls_implicit)
-            .credentials((username.unwrap(), password.unwrap()));
-        Some(Ok(smtp))
-    } else {
-        None
-    }
+    pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+    pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+    pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 }
 
 pub async fn run_server(
@@ -153,7 +115,7 @@ pub async fn run_server(
     port_tx: tokio::sync::oneshot::Sender<u16>,
 ) -> anyhow::Result<()> {
     if let Some(mut rsmq) = rsmq.clone() {
-        for tag in ALL_TAGS.clone() {
+        for tag in ALL_TAGS.read().await.iter() {
             let r =
                 rsmq_async::RsmqConnection::create_queue(&mut rsmq, &tag, None, None, None).await;
             if let Err(e) = r {
@@ -185,7 +147,9 @@ pub async fn run_server(
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
-        .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
+        .layer(DefaultBodyLimit::max(
+            REQUEST_SIZE_LIMIT.read().await.clone(),
+        ));
 
     let cors = CorsLayer::new()
         .allow_methods([Method::GET, Method::POST])
@@ -239,6 +203,7 @@ pub async fn run_server(
                 .nest("/settings", settings::global_service())
                 .nest("/jobs", jobs::global_root_service())
                 .nest("/workers", workers::global_service())
+                .nest("/configs", configs::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/groups", groups::global_service())
                 .nest("/flows", flows::global_service())
@@ -287,15 +252,16 @@ pub async fn run_server(
 
     let instance_name = rd_string(5);
 
-    tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening");
     let server = axum::Server::bind(&addr).serve(app.into_make_service());
 
     let port = server.local_addr().port();
     tracing::info!(
+        instance = %instance_name,
         "server started on port={} and addr={}",
         port,
         server.local_addr().ip()
     );
+
     port_tx
         .send(server.local_addr().port())
         .expect("Failed to send port");
@@ -305,8 +271,6 @@ pub async fn run_server(
         println!("Graceful shutdown of server");
     });
 
-    tokio::spawn(async move { auth_cache.monitor().await });
-
     server.await?;
     Ok(())
 }
@@ -354,13 +318,7 @@ async fn ee_license() -> &'static str {
 
 #[cfg(feature = "enterprise")]
 async fn ee_license() -> String {
-    LICENSE_KEY
-        .as_ref()
-        .unwrap()
-        .split(".")
-        .next()
-        .unwrap()
-        .to_string()
+    LICENSE_KEY_ID.read().await.clone()
 }
 
 async fn openapi() -> &'static str {

backend/windmill-api/src/oauth2.rs

@@ -81,7 +81,7 @@ pub fn workspaced_service() -> Router {
         .route("/connect_slack_callback", post(connect_slack_callback))
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ClientWithScopes {
     client: OClient,
     scopes: Vec<String>,
@@ -120,7 +120,10 @@ pub struct AllClients {
     pub slack: Option<OClient>,
 }
 
-pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
+pub fn build_oauth_clients(
+    base_url: &str,
+    oauths_from_config: Option<HashMap<String, OAuthClient>>,
+) -> anyhow::Result<AllClients> {
     let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
         "../../oauth_connect.json"
     ))?;
@@ -128,26 +131,36 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         "../../oauth_login.json"
     ))?;
 
-    let path = "./oauth.json";
-    let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
-        str::from_utf8(
-            &base64::engine::general_purpose::STANDARD
-                .decode(e)
-                .map_err(to_anyhow)?,
-        )?
-        .to_string()
-    } else if std::path::Path::new(path).exists() {
-        fs::read_to_string(path).map_err(to_anyhow)?
+    let oauths = if let Some(oauths) = oauths_from_config {
+        oauths
     } else {
-        tracing::warn!("oauth.json not found, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
+        let path = "./oauth.json";
+        let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
+            str::from_utf8(
+                &base64::engine::general_purpose::STANDARD
+                    .decode(e)
+                    .map_err(to_anyhow)?,
+            )?
+            .to_string()
+        } else if std::path::Path::new(path).exists() {
+            fs::read_to_string(path).map_err(to_anyhow)?
+        } else {
+            tracing::warn!("oauth.json not found, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
 
-    if content.is_empty() {
-        tracing::warn!("oauth.json is empty, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
-    let oauths: HashMap<String, OAuthClient> =
+        if content.is_empty() {
+            tracing::warn!("oauth.json is empty, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
         match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
             Ok(clients) => clients,
             Err(e) => {
@@ -156,7 +169,8 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             }
         }
         .into_iter()
-        .collect();
+        .collect()
+    };
 
     tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", "));
 
@@ -168,7 +182,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -177,17 +191,25 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 base_url,
                 None,
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: client_params.allowed_domains.clone(),
-                    userinfo_url: config.userinfo_url,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: client_params.allowed_domains.clone(),
+                            userinfo_url: config.userinfo_url,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
@@ -199,7 +221,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -212,43 +234,61 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                     None
                 },
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: None,
-                    userinfo_url: None,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: None,
+                            userinfo_url: None,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
-    let slack = oauths.get("slack").map(|v| {
-        build_basic_client(
-            "slack".to_string(),
-            OAuthConfig {
-                auth_url: "https://slack.com/oauth/authorize".to_string(),
-                token_url: "https://slack.com/api/oauth.access".to_string(),
-                userinfo_url: None,
-                scopes: None,
-                extra_params: None,
-                extra_params_callback: None,
-                req_body_auth: None,
-            },
-            v.clone(),
-            false,
-            base_url,
-            Some(format!("{base_url}/oauth/callback_slack")),
-        )
-        .1
-    });
-
-    Ok(AllClients { logins, connects, slack })
+    let slack = oauths
+        .get("slack")
+        .map(|v| {
+            build_basic_client(
+                "slack".to_string(),
+                OAuthConfig {
+                    auth_url: "https://slack.com/oauth/authorize".to_string(),
+                    token_url: "https://slack.com/api/oauth.access".to_string(),
+                    userinfo_url: None,
+                    scopes: None,
+                    extra_params: None,
+                    extra_params_callback: None,
+                    req_body_auth: None,
+                },
+                v.clone(),
+                false,
+                base_url,
+                Some(format!("{base_url}/oauth/callback_slack")),
+            )
+            .map(|x| x.1)
+            .map_err(|e| {
+                tracing::error!("Error building oauth slack client: {e}");
+                e
+            })
+            .ok()
+        })
+        .flatten();
+    let all_clients = AllClients { logins, connects, slack };
+    tracing::info!("Final oauth config: {all_clients:#?}");
+    Ok(all_clients)
 }
 
+use anyhow::anyhow;
 pub fn build_basic_client(
     name: String,
     config: OAuthConfig,
@@ -256,9 +296,11 @@ pub fn build_basic_client(
     login: bool,
     base_url: &str,
     override_callback: Option<String>,
-) -> (String, OClient) {
-    let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
-    let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL");
+) -> error::Result<(String, OClient)> {
+    let auth_url = Url::parse(&config.auth_url)
+        .map_err(|e| anyhow!("Invalid authorization endpoint URL: {e}"))?;
+    let token_url =
+        Url::parse(&config.token_url).map_err(|e| anyhow!("Invalid token endpoint URL: {e}"))?;
 
     let redirect_url = if login {
         format!("{base_url}/user/login_callback/{name}")
@@ -273,9 +315,12 @@ pub fn build_basic_client(
         client.set_auth_type(AuthType::RequestBody);
     }
     client.set_client_secret(client_params.secret.clone());
-    client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL"));
+    client.set_redirect_url(
+        Url::parse(&redirect_url).map_err(|e| anyhow!("Invalid redirect URL: {e}"))?,
+    );
+
     // Set up the config for the Github OAuth2 process.
-    (name.to_string(), client)
+    Ok((name.to_string(), client))
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -314,7 +359,7 @@ async fn connect(
     cookies: Cookies,
 ) -> error::Result<Redirect> {
     let mut query = query.clone();
-    let connects = &OAUTH_CLIENTS.connects;
+    let connects = &OAUTH_CLIENTS.read().await.connects;
     let scopes = query
         .get("scopes")
         .map(|x| x.split('+').map(|x| x.to_owned()).collect());
@@ -330,7 +375,7 @@ async fn connect(
         cookies,
         scopes,
         extra_params,
-        *IS_SECURE,
+        IS_SECURE.read().await.clone(),
     )
 }
 
@@ -341,6 +386,7 @@ struct CreateAccount {
     refresh_token: Option<String>,
     expires_in: i64,
 }
+
 async fn create_account(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -405,6 +451,8 @@ struct Logins {
 async fn list_logins(Extension(sso): Extension<Arc<SamlSsoLogin>>) -> error::JsonResult<Logins> {
     Ok(Json(Logins {
         oauth: OAUTH_CLIENTS
+            .read()
+            .await
             .logins
             .keys()
             .map(|x| x.to_owned())
@@ -420,7 +468,7 @@ struct ScopesAndParams {
 }
 async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
     Ok(Json(
-        (&OAUTH_CLIENTS.connects)
+        (&OAUTH_CLIENTS.read().await.connects)
             .into_iter()
             .map(|(k, v)| {
                 (
@@ -437,6 +485,8 @@ async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>>
 
 async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     let mut client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -452,7 +502,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     client.add_scope("commands");
     let url = client.authorize_url(&state);
 
-    set_cookie(&state, cookies, *IS_SECURE);
+    set_cookie(&state, cookies, IS_SECURE.read().await.clone());
     Ok(Redirect::to(url.as_str()))
 }
 
@@ -495,8 +545,15 @@ async fn disconnect_slack(
 }
 
 async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
-    let clients = &OAUTH_CLIENTS.logins;
-    oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
+    let clients = &OAUTH_CLIENTS.read().await.logins;
+    oauth_redirect(
+        clients,
+        client_name,
+        cookies,
+        None,
+        None,
+        IS_SECURE.read().await.clone(),
+    )
 }
 
 #[derive(Deserialize)]
@@ -531,6 +588,8 @@ pub async fn _refresh_token<'c>(
     .await?;
     let account = not_found_if_none(account, "Account", &id.to_string())?;
     let client = (&OAUTH_CLIENTS
+        .read()
+        .await
         .connects
         .get(&account.client)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
@@ -621,11 +680,10 @@ async fn connect_callback(
     Path(client_name): Path<String>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::JsonResult<TokenResponse> {
-    let client_w_scopes = OAUTH_CLIENTS
-        .connects
+    let connects = &OAUTH_CLIENTS.read().await.connects;
+    let client_w_scopes = connects
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
-
     let client = client_w_scopes.client.to_owned();
     let extra_params = client_w_scopes.extra_params_callback.clone();
     let token_response =
@@ -643,6 +701,8 @@ async fn connect_slack_callback(
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
     let client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -857,7 +917,7 @@ async fn slack_command(
                 None,
             )
             .await?;
-            let url = BASE_URL.to_owned();
+            let url = BASE_URL.read().await.clone();
             tx.commit().await?;
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
@@ -879,10 +939,13 @@ async fn login_callback(
     Extension(webhook): Extension<WebhookShared>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
-    let client_w_config = &OAUTH_CLIENTS
-        .logins
-        .get(&client_name)
-        .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
+    let client_w_config = {
+        let clients = OAUTH_CLIENTS.read().await.logins.clone();
+        clients
+            .get(&client_name)
+            .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
+            .clone()
+    };
     let client = client_w_config.client.to_owned();
     let token_res =
         exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;

backend/windmill-api/src/saml.rs

@@ -71,7 +71,7 @@ pub async fn build_sp_extension() -> anyhow::Result<(ServiceProviderExt, SamlSso
                 ..ContactPerson::default()
             })
             .idp_metadata(idp_metadata)
-            .acs_url(format!("{}/api/saml/acs", *BASE_URL))
+            .acs_url(format!("{}/api/saml/acs", BASE_URL.read().await.clone()))
             .build()?;
 
         tracing::info!("SAML Configured, sso login link at: {:?}", url);

backend/windmill-api/src/schedule.rs

@@ -97,6 +97,21 @@ async fn create_schedule(
     let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
+    #[cfg(not(feature = "enterprise"))]
+    if ns.on_recovery.is_some() {
+        return Err(Error::BadRequest(
+            "on_recovery is only available in enterprise version".to_string(),
+        ));
+    }
+
+    #[cfg(not(feature = "enterprise"))]
+    if ns.on_failure_times.is_some() && ns.on_failure_times.unwrap() > 1 {
+        return Err(Error::BadRequest(
+            "on_failure with a number of times > 1 is only available in enterprise version"
+                .to_string(),
+        ));
+    }
+
     cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
     check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
     check_flow_conflict(

backend/windmill-api/src/scripts.rs

@@ -19,7 +19,7 @@ use axum::{
     Json, Router,
 };
 use hyper::StatusCode;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::json;
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
@@ -41,7 +41,8 @@ use windmill_common::{
     },
     users::username_to_permissioned_as,
     utils::{
-        list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
+        list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
+        Pagination, StripPath,
     },
 };
 use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
@@ -80,6 +81,7 @@ pub fn global_service() -> Router {
         .route("/hub/list", get(list_hub_scripts))
         .route("/hub/get/*path", get(get_hub_script_by_path))
         .route("/hub/get_full/*path", get(get_full_hub_script_by_path))
+        .route("/hub/query", get(query_hub_scripts))
 }
 
 pub fn global_unauthed_service() -> Router {
@@ -209,6 +211,28 @@ async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serd
     Ok(Json(asks))
 }
 
+#[derive(Deserialize)]
+struct HubScriptsQuery {
+    text: String,
+    kind: Option<String>,
+    limit: Option<i64>,
+}
+async fn query_hub_scripts(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Query(query): Query<HubScriptsQuery>,
+) -> JsonResult<serde_json::Value> {
+    let asks = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/scripts/query",
+        &email,
+        &query.text,
+        &query.kind,
+        &query.limit,
+    )
+    .await?;
+    Ok(Json(asks))
+}
+
 fn hash_script(ns: &NewScript) -> i64 {
     let mut dh = DefaultHasher::new();
     ns.hash(&mut dh);

backend/windmill-api/src/settings.rs

@@ -6,8 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::time::Duration;
+
 use crate::{
     db::{ApiAuthed, DB},
+    ee::validate_license_key,
     utils::require_super_admin,
 };
 
@@ -17,18 +20,74 @@ use axum::{
     Json, Router,
 };
 
+use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
+use serde::Deserialize;
+use tokio::time::timeout;
 use windmill_common::{
-    error::{self, JsonResult},
+    error::{self, to_anyhow, JsonResult},
     global_settings::ENV_SETTINGS,
+    server::Smtp,
 };
 
 pub fn global_service() -> Router {
     Router::new()
-        .route("/local", get(get_local_settings))
+        .route("/envs", get(get_local_settings))
         .route(
             "/global/:key",
             post(set_global_setting).get(get_global_setting),
         )
+        .route("/test_smtp", post(test_email))
+        .route("/test_license_key", post(test_license_key))
+}
+
+#[derive(Deserialize)]
+pub struct TestEmail {
+    pub to: String,
+    pub smtp: Smtp,
+}
+
+pub async fn test_email(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(test_email): Json<TestEmail>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    let smtp = test_email.smtp;
+    let to = test_email.to;
+    let client = SmtpClientBuilder::new(smtp.host, smtp.port)
+        .implicit_tls(smtp.tls_implicit)
+        .credentials((smtp.username, smtp.password));
+    let message = MessageBuilder::new()
+        .from(("Windmill", smtp.from.as_str()))
+        .to(to.clone())
+        .subject("Test email from Windmill")
+        .text_body("Test email content");
+    let dur = Duration::from_secs(3);
+    timeout(dur, client.connect())
+        .await
+        .map_err(to_anyhow)?
+        .map_err(to_anyhow)?
+        .send(message)
+        .await
+        .map_err(to_anyhow)?;
+    tracing::info!("Sent test email to {to}");
+    Ok("Sent test email".to_string())
+}
+
+#[derive(Deserialize)]
+pub struct TestKey {
+    pub license_key: String,
+}
+
+
+pub async fn test_license_key(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(TestKey { license_key }): Json<TestKey>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    validate_license_key(license_key).await?;
+    Ok("Sent test email".to_string())
 }
 
 pub async fn get_local_settings(

backend/windmill-api/src/users.rs

@@ -8,7 +8,7 @@
 
 #![allow(non_snake_case)]
 
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
 use crate::db::ApiAuthed;
 
@@ -18,7 +18,7 @@ use crate::{
     utils::require_super_admin,
     webhook_util::{InstanceEvent, WebhookShared},
     workspaces::invite_user_to_all_auto_invite_worspaces,
-    BASE_URL, COOKIE_DOMAIN, IS_SECURE, SMTP_CLIENT, SMTP_FROM,
+    BASE_URL, COOKIE_DOMAIN, IS_SECURE,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
@@ -32,24 +32,25 @@ use axum::{
 use hyper::{header::LOCATION, StatusCode};
 use lazy_static::lazy_static;
 use mail_send::mail_builder::MessageBuilder;
+use mail_send::SmtpClientBuilder;
+use quick_cache::sync::Cache;
 use rand::rngs::OsRng;
 use regex::Regex;
-use retainer::Cache;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use time::OffsetDateTime;
 use tower_cookies::{Cookie, Cookies};
 use tracing::{Instrument, Span};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::users::truncate_token;
+use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG};
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
     users::SUPERADMIN_SECRET_EMAIL,
     utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
 };
-use windmill_queue::CLOUD_HOSTED;
 
-const TTL_TOKEN_CACHE_S: u64 = 60; // 60s
 pub const TTL_TOKEN_DB_H: u32 = 72;
 
 const COOKIE_NAME: &str = "token";
@@ -102,19 +103,24 @@ pub fn make_unauthed_service() -> Router {
         .route("/logout", get(logout))
 }
 
+#[derive(Clone)]
+pub struct ExpiringAuthCache {
+    pub authed: ApiAuthed,
+    pub expiry: Option<chrono::DateTime<chrono::Utc>>,
+}
 pub struct AuthCache {
-    cache: Cache<(String, String), ApiAuthed>,
+    cache: Cache<(String, String), ExpiringAuthCache>,
     db: DB,
     superadmin_secret: Option<String>,
 }
 
 impl AuthCache {
     pub fn new(db: DB, superadmin_secret: Option<String>) -> Self {
-        AuthCache { cache: Cache::new(), db, superadmin_secret }
+        AuthCache { cache: Cache::new(300), db, superadmin_secret }
     }
 
     pub async fn invalidate(&self, w_id: &str, token: String) {
-        self.cache.remove(&(w_id.to_string(), token)).await;
+        self.cache.remove(&(w_id.to_string(), token));
     }
 
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
@@ -122,13 +128,17 @@ impl AuthCache {
             w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
             token.to_string(),
         );
-        let s = self.cache.get(&key).await.map(|c| c.to_owned());
+        let s = self.cache.get(&key).map(|c| c.to_owned());
         match s {
-            a @ Some(_) => a,
-            None => {
-                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>)>(
+            Some(ExpiringAuthCache { authed, expiry })
+                if expiry.is_none() || expiry.unwrap() > chrono::Utc::now() =>
+            {
+                Some(authed)
+            }
+            _ => {
+                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>, Option<chrono::DateTime<chrono::Utc>>)>(
                     "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
-                     OR expiration IS NULL) RETURNING owner, email, super_admin, scopes",
+                     OR expiration IS NULL) RETURNING owner, email, super_admin, scopes, expiration",
                 )
                 .bind(token)
                 .fetch_optional(&self.db)
@@ -139,7 +149,7 @@ impl AuthCache {
                 if let Some(user) = user_o {
                     let authed_o = {
                         match user {
-                            (Some(owner), Some(email), super_admin, _) if w_id.is_some() => {
+                            (Some(owner), Some(email), super_admin, _, _) if w_id.is_some() => {
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
                                         let (is_admin, is_operator) = if super_admin {
@@ -218,7 +228,7 @@ impl AuthCache {
                                     })
                                 }
                             }
-                            (_, Some(email), super_admin, scopes) => {
+                            (_, Some(email), super_admin, scopes, _) => {
                                 if w_id.is_some() {
                                     let row_o = sqlx::query_as::<_, (String, bool, bool)>(
                                         "SELECT username, is_admin, operator FROM usr where email = $1 AND \
@@ -288,9 +298,10 @@ impl AuthCache {
                         }
                     };
                     if let Some(authed) = authed_o.as_ref() {
-                        self.cache
-                            .insert(key, authed.clone(), Duration::from_secs(TTL_TOKEN_CACHE_S))
-                            .await;
+                        self.cache.insert(
+                            key,
+                            ExpiringAuthCache { authed: authed.clone(), expiry: user.4 },
+                        );
                     }
                     authed_o
                 } else if self
@@ -314,10 +325,6 @@ impl AuthCache {
             }
         }
     }
-
-    pub async fn monitor(&self) {
-        self.cache.monitor(20, 0.25, Duration::from_secs(10)).await;
-    }
 }
 
 async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
@@ -712,7 +719,7 @@ async fn list_users(
         SELECT usr.*, usage.*
           FROM usr
              , LATERAL (
-                SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0) executions
+                SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions
                   FROM completed_job
                  WHERE workspace_id = $1
                    AND job_kind NOT IN ('flow', 'flowpreview')
@@ -1219,13 +1226,7 @@ async fn decline_invite(
 }
 
 lazy_static! {
-    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
-
-    pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
-    .ok()
-    .and_then(|x| x.parse::<u32>().ok())
-    .unwrap_or(60 * 60 * 24 * 60); // 60 days
-
+    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap();
 }
 
 async fn accept_invite(
@@ -1236,7 +1237,7 @@ async fn accept_invite(
 ) -> Result<(StatusCode, String)> {
     if !VALID_USERNAME.is_match(&nu.username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1319,7 +1320,7 @@ async fn add_user_to_workspace<'c>(
 
     if !VALID_USERNAME.is_match(username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1570,7 +1571,9 @@ async fn create_user(
 Log in and change your password: {}/user/login?email={}&password={}&rd=%2F%23user-settings
 
 You can then join or create a workspace. Happy building!",
-            *BASE_URL, &nu.email, &nu.password
+            BASE_URL.read().await.clone(),
+            &nu.email,
+            &nu.password
         ),
         &nu.email,
     );
@@ -1590,13 +1593,17 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
 }
 
 pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &str) -> Result<()> {
-    if let Some(ref smtp) = *SMTP_CLIENT {
+    if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() {
+        let client = SmtpClientBuilder::new(smtp.host, smtp.port)
+            .implicit_tls(smtp.tls_implicit)
+            .credentials((smtp.username, smtp.password));
         let message = MessageBuilder::new()
-            .from(("Windmill", SMTP_FROM.as_str()))
+            .from(("Windmill", smtp.from.as_str()))
             .to(to)
             .subject(subject)
             .text_body(content);
-        smtp.connect()
+        client
+            .connect()
             .await
             .map_err(to_anyhow)?
             .send(message)
@@ -1942,7 +1949,7 @@ pub async fn create_session_token<'c>(
     .execute(&mut **tx)
     .await?;
     let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
-    cookie.set_secure(*IS_SECURE);
+    cookie.set_secure(IS_SECURE.read().await.clone());
     cookie.set_same_site(Some(cookie::SameSite::Lax));
     cookie.set_http_only(true);
     cookie.set_path(COOKIE_PATH);
@@ -2240,6 +2247,20 @@ pub struct LoginUserInfo {
     pub displayName: Option<String>,
 }
 
+async fn _check_nb_of_user(db: &DB) -> Result<()> {
+    let nb_groups =
+        sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)
+            .fetch_one(db)
+            .await?;
+    if nb_groups.unwrap_or(0) >= 50 {
+        return Err(Error::BadRequest(
+            "You have reached the maximum number of oauth users accounts (50) without an enterprise license"
+                .to_string(),
+        ));
+    }
+    return Ok(());
+}
+
 pub async fn login_externally(
     db: DB,
     email: &String,
@@ -2292,6 +2313,10 @@ pub async fn login_externally(
         if (name.is_none() || name == Some(String::new())) && user.is_some() {
             name = user.clone().unwrap().displayName;
         }
+
+        #[cfg(not(feature = "enterprise"))]
+        _check_nb_of_user(&db).await?;
+
         sqlx::query(&format!(
             "INSERT INTO password (email, name, company, login_type, verified) VALUES ($1, \
                  $2, $3, '{}', true)",
@@ -2340,78 +2365,3 @@ pub async fn login_externally(
     tx.commit().await?;
     Ok(())
 }
-
-pub async fn delete_expired_items_perdiodically(
-    db: &DB,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-) -> () {
-    loop {
-        let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-            "DELETE FROM token WHERE expiration <= now()
-        RETURNING concat(substring(token for 10), '*****')",
-        )
-        .fetch_all(db)
-        .await;
-
-        match tokens_deleted_r {
-            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
-            Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
-        }
-
-        let pip_resolution_r = sqlx::query_scalar!(
-            "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
-        )
-        .fetch_all(db)
-        .await;
-
-        match pip_resolution_r {
-            Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
-            Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
-        }
-
-        let deleted_cache = sqlx::query_scalar!(
-            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
-        )
-        .fetch_all(db)
-        .await;
-
-        match deleted_cache {
-            Ok(res) => tracing::debug!("deleted {} cache resource: {:?}", res.len(), res),
-            Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
-        }
-
-        if *JOB_RETENTION_SECS > 0 {
-            let deleted_jobs = sqlx::query_scalar!(
-                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
-                *JOB_RETENTION_SECS as i64
-            )
-            .fetch_all(db)
-            .await;
-
-            match deleted_jobs {
-                Ok(deleted_jobs) => {
-                    tracing::info!(
-                        "deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
-                        deleted_jobs.len(),
-                        deleted_jobs
-                    )
-                }
-                Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
-            }
-        }
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(600))     => (),
-            _ = rx. recv() => {
-                 println!("received killpill for delete expired tokens");
-                 break;
-            }
-        }
-    }
-}
-
-pub fn truncate_token(token: &str) -> String {
-    let mut s = token[..10].to_owned();
-    s.push_str("*****");
-    s
-}

backend/windmill-api/src/variables.rs

@@ -68,6 +68,7 @@ async fn list_contextual_variables(
             Some("u/user/triggering_flow_path".to_string()),
             Some("c".to_string()),
         )
+        .await
         .to_vec(),
     ))
 }

backend/windmill-api/src/webhook_util.rs

@@ -1,7 +1,8 @@
 use std::time::Duration;
 
+use quick_cache::sync::Cache;
 use serde::Serialize;
-use tokio::{select, sync::mpsc, time::interval};
+use tokio::{select, sync::mpsc};
 use windmill_common::METRICS_ENABLED;
 
 use crate::db::DB;
@@ -78,8 +79,7 @@ impl WebhookShared {
                 .timeout(Duration::from_secs(5))
                 .build()
                 .unwrap();
-            let cache = retainer::Cache::new();
-            let mut cache_purge_interval = interval(Duration::from_secs(30));
+            let cache = Cache::new(100);
 
             loop {
                 select! {
@@ -87,7 +87,7 @@ impl WebhookShared {
                     _ = shutdown_rx.recv() => break,
                     r = rx.recv() => match r {
                         Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
-                            let url_guard = match cache.get(&workspace_id).await {
+                            let webhook_opt = match cache.get(&workspace_id) {
                                 Some(guard) => {
                                     guard
                                 },
@@ -104,16 +104,14 @@ impl WebhookShared {
                                             tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}");
                                             continue;
                                         };
-                                    cache.insert(workspace_id.clone(), webook_opt, Duration::from_secs(30)).await;
-                                    cache.get(&workspace_id).await.unwrap()
+                                    cache.insert(workspace_id, webook_opt.clone());
+                                    webook_opt
                                 }
                             };
-                            let webook_opt = url_guard.value();
-                            if let Some(url) = webook_opt {
+                            if let Some(url) = webhook_opt {
                                 let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
                                 let _ = client.post(url).json(&message).send().await;
                                 timer.map(|x| x.stop_and_record());
-                                drop(url_guard);
                             }
                         },
                         Some(WebhookPayload::InstanceEvent(event)) => {
@@ -125,10 +123,6 @@ impl WebhookShared {
                         },
                         None => break,
                     },
-                    _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
-                        tracing::trace!("Purging Webhook Cache");
-                        cache.purge(10, 0.50).await;
-                    },
                 }
             }
         });

backend/windmill-api/src/workers.rs

@@ -12,71 +12,23 @@ use axum::{
     Json, Router,
 };
 
-use itertools::Itertools;
-use regex::Regex;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use windmill_common::{
     db::UserDB,
     error::JsonResult,
     utils::{paginate, Pagination},
+    worker::ALL_TAGS,
 };
 
-use std::collections::HashMap;
-#[cfg(feature = "benchmark")]
-use std::sync::atomic::Ordering;
-#[cfg(feature = "benchmark")]
-use windmill_queue::IDLE_WORKERS;
-
 use crate::db::ApiAuthed;
 
-#[cfg(not(feature = "benchmark"))]
 pub fn global_service() -> Router {
     Router::new()
         .route("/list", get(list_worker_pings))
         .route("/custom_tags", get(get_custom_tags))
 }
 
-#[cfg(feature = "benchmark")]
-pub fn global_service() -> Router {
-    Router::new()
-        .route("/toggle", get(toggle))
-        .route("/list", get(list_worker_pings))
-        .route("/custom_tags", get(get_custom_tags))
-}
-
-lazy_static::lazy_static! {
-    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
-        .ok()
-        .map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
-
-    pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec<String>, HashMap<String, Vec<String>>) =  process_custom_tags(std::env::var("CUSTOM_TAGS")
-        .ok());
-
-    pub static ref ALL_TAGS: Vec<String> = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat();
-
-}
-
-fn process_custom_tags(o: Option<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
-    let regex = Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
-    if let Some(s) = o {
-        let mut global = vec![];
-        let mut specific: HashMap<String, Vec<String>> = HashMap::new();
-        for e in s.split(",") {
-            if let Some(cap) = regex.captures(e) {
-                let tag = cap.get(1).unwrap().as_str().to_string();
-                let workspaces = cap.get(2).unwrap().as_str().split("+");
-                specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
-            } else {
-                global.push(e.to_string());
-            }
-        }
-        (global, specific)
-    } else {
-        (vec![], HashMap::new())
-    }
-}
-
 #[derive(FromRow, Serialize, Deserialize)]
 struct WorkerPing {
     worker: String,
@@ -86,6 +38,7 @@ struct WorkerPing {
     ip: String,
     jobs_executed: i32,
     custom_tags: Option<Vec<String>>,
+    worker_group: String,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -104,7 +57,7 @@ async fn list_worker_pings(
 
     let rows = sqlx::query_as!(
         WorkerPing,
-        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
         per_page as i64,
         offset as i64
     )
@@ -114,12 +67,6 @@ async fn list_worker_pings(
     Ok(Json(rows))
 }
 
-#[cfg(feature = "benchmark")]
-async fn toggle(Query(query): Query<EnableWorkerQuery>) -> JsonResult<bool> {
-    IDLE_WORKERS.store(query.disable, Ordering::Relaxed);
-    Ok(Json(IDLE_WORKERS.load(Ordering::Relaxed)))
-}
-
 async fn get_custom_tags() -> Json<Vec<String>> {
-    Json(ALL_TAGS.clone())
+    Json(ALL_TAGS.read().await.clone().into())
 }

backend/windmill-api/src/workspaces.rs

@@ -74,13 +74,19 @@ pub fn workspaced_service() -> Router {
         .route("/edit_error_handler", post(edit_error_handler));
 
     #[cfg(feature = "enterprise")]
-    tracing::info!("stripe enabled");
-
-    #[cfg(feature = "enterprise")]
-    let router = router
-        .route("/checkout", get(stripe_checkout))
-        .route("/billing_portal", get(stripe_portal));
+    { 
+        if std::env::var("STRIPE_KEY").is_err() {
+            return router;
+        } else {
+            tracing::info!("stripe enabled");
+            
+            return router
+            .route("/checkout", get(stripe_checkout))
+            .route("/billing_portal", get(stripe_portal));
+        }
+    }
 
+    #[cfg(not(feature = "enterprise"))]
     router
 }
 pub fn global_service() -> Router {
@@ -278,8 +284,9 @@ async fn stripe_checkout(
         require_admin(authed.is_admin, &authed.username)?;
 
         let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-        let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL);
-        let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL);
+        let base_url = BASE_URL.read().await.clone();
+        let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url);
+        let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url);
         let checkout_session = {
             let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd);
             params.mode = Some(stripe::CheckoutSessionMode::Subscription);
@@ -326,7 +333,7 @@ async fn stripe_portal(
     .await?
     .ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?;
     let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-    let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL);
+    let success_rd = format!("{}/workspace_settings?tab=premium", BASE_URL.read().await.clone());
     let portal_session = {
         let customer_id = CustomerId::from_str(&customer_id).unwrap();
         let mut params = stripe::CreateBillingPortalSession::new(customer_id);
@@ -839,10 +846,24 @@ async fn check_name_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str)
 
 lazy_static::lazy_static! {
 
-    pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(false));
+    pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(true));
 
 }
 
+async fn _check_nb_of_workspaces(db: &DB) -> Result<()> {
+    let nb_workspaces = sqlx::query_scalar!("SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",)
+        .fetch_one(db)
+        .await?;
+    if nb_workspaces.unwrap_or(0) >= 3 {
+        return Err(Error::BadRequest(
+            "You have reached the maximum number of workspaces (3 outside of default group 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
+                .to_string(),
+        ));
+    }
+    return Ok(());
+}
+
+
 async fn create_workspace(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -852,6 +873,10 @@ async fn create_workspace(
     if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN {
         require_super_admin(&db, &authed.email).await?;
     }
+
+    #[cfg(not(feature = "enterprise"))]
+    _check_nb_of_workspaces(&db).await?;
+
     let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
     check_name_conflict(&mut tx, &nw.id).await?;
@@ -926,6 +951,21 @@ async fn create_workspace(
     .execute(&mut *tx)
     .await?;
 
+    sqlx::query!(
+        "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+        nw.id,
+    )
+    .execute(&mut *tx)
+    .await?;
+
+
+    sqlx::query!(
+        "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+        nw.id,
+    )
+    .execute(&mut *tx)
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed.username,
@@ -1070,6 +1110,10 @@ async fn delete_workspace(
         .execute(&mut *tx)
         .await?;
 
+    sqlx::query!("DELETE FROM resource_type WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
     sqlx::query!(
         "DELETE FROM workspace_invite WHERE workspace_id = $1",
         &w_id
@@ -1178,7 +1222,7 @@ async fn invite_user(
             "You have been granted access to Windmill's workspace {w_id}
 
 If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
-            *BASE_URL
+            BASE_URL.read().await.clone()
         ),
         &nu.email,
     );
@@ -1207,7 +1251,7 @@ async fn add_user(
     let mut tx = db.begin().await?;
     if !VALID_USERNAME.is_match(&nu.username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1242,7 +1286,7 @@ async fn add_user(
             "You have been granted access to Windmill's workspace {w_id} by {email}
 
 If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
-            *BASE_URL
+            BASE_URL.read().await.clone()
         ),
         &nu.email,
     );

backend/windmill-common/Cargo.toml

@@ -44,3 +44,5 @@ reqwest = { workspace = true, optional = true }
 tracing-subscriber = { workspace = true, optional = true }
 lazy_static.workspace = true
 tracing-flame = { version = "^0", optional = true }
+itertools.workspace = true
+regex.workspace = true

backend/windmill-common/src/global_settings.rs

@@ -1,4 +1,10 @@
 pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync";
+pub const CUSTOM_TAGS_SETTING: &str = "custom_tags";
+pub const BASE_URL_SETTING: &str = "base_url";
+pub const OAUTH_SETTING: &str = "oauths";
+pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
+pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
+pub const LICENSE_KEY_SETTING: &str = "license_key";
 
 pub const ENV_SETTINGS: [&str; 54] = [
     "DISABLE_NSJAIL",
@@ -41,8 +47,6 @@ pub const ENV_SETTINGS: [&str; 54] = [
     "INSTANCE_EVENTS_WEBHOOK",
     "CLOUD_HOSTED",
     "GLOBAL_CACHE_INTERVAL",
-    "WORKER_TAGS",
-    "CUSTOM_TAGS",
     "JOB_RETENTION_SECS",
     "WAIT_RESULT_FAST_POLL_DURATION_SECS",
     "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
@@ -55,4 +59,6 @@ pub const ENV_SETTINGS: [&str; 54] = [
     "SMTP_TLS_IMPLICIT",
     "CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
     "GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
+    "MAX_WAIT_FOR_SIGTERM",
+    "WORKER_GROUP",
 ];

backend/windmill-common/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::{net::SocketAddr, sync::Arc};
 
 use error::Error;
 use scripts::ScriptLang;
@@ -24,9 +24,11 @@ pub mod more_serde;
 pub mod oauth2;
 pub mod schedule;
 pub mod scripts;
+pub mod server;
 pub mod users;
 pub mod utils;
 pub mod variables;
+pub mod worker;
 
 #[cfg(feature = "tracing_init")]
 pub mod tracing_init;
@@ -46,7 +48,7 @@ lazy_static::lazy_static! {
     .flatten()
     .flatten();
     pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
-    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+    pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
     pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 }
 
@@ -77,12 +79,16 @@ pub async fn shutdown_signal(
     Ok(())
 }
 
+use tokio::sync::RwLock;
+#[cfg(feature = "prometheus")]
+use tokio::task::JoinHandle;
+
 #[cfg(feature = "prometheus")]
 pub async fn serve_metrics(
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
     ready_worker_endpoint: bool,
-) -> Result<(), hyper::Error> {
+) -> JoinHandle<()> {
     use std::sync::atomic::Ordering;
 
     use axum::{routing::get, Router};
@@ -104,13 +110,18 @@ pub async fn serve_metrics(
         router
     };
 
-    axum::Server::bind(&addr)
-        .serve(router.into_make_service())
-        .with_graceful_shutdown(async {
-            rx.recv().await.ok();
-            println!("Graceful shutdown of metrics");
-        })
-        .await
+    tokio::spawn(async move {
+        if let Err(e) = axum::Server::bind(&addr)
+            .serve(router.into_make_service())
+            .with_graceful_shutdown(async {
+                rx.recv().await.ok();
+                println!("Graceful shutdown of metrics");
+            })
+            .await
+        {
+            tracing::error!("Error serving metrics: {}", e);
+        }
+    })
 }
 
 async fn metrics() -> Result<String, Error> {

backend/windmill-common/src/scripts.rs

@@ -161,6 +161,7 @@ pub struct Script {
     pub concurrent_limit: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_time_window_s: Option<i32>,
+    pub dedicated_worker: Option<bool>,
 }
 
 #[derive(Serialize)]
@@ -274,6 +275,7 @@ pub async fn get_hub_script_by_path(
         &format!("https://hub.windmill.dev/raw/{path}.ts"),
         email,
         true,
+        None,
     )
     .await?
     .text()
@@ -303,6 +305,7 @@ pub async fn get_full_hub_script_by_path(
         &format!("https://hub.windmill.dev/raw2/{path}"),
         email,
         true,
+        None,
     )
     .await?
     .json::<HubScript>()

backend/windmill-common/src/server.rs

@@ -0,0 +1,112 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{error, DB};
+
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+pub struct Smtp {
+    pub host: String,
+    pub username: String,
+    pub password: String,
+    pub port: u16,
+    pub from: String,
+    pub tls_implicit: bool,
+}
+
+#[derive(Serialize, Deserialize, PartialEq)]
+pub struct ServerConfigOpt {
+    pub smtp_host: Option<String>,
+    pub smtp_username: Option<String>,
+    pub smtp_password: Option<String>,
+    pub smtp_port: Option<u16>,
+    pub smtp_from: Option<String>,
+    pub smtp_tls_implicit: Option<bool>,
+    pub timeout_wait_result: Option<u64>,
+}
+
+pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
+    let config: ServerConfigOpt =
+        sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",)
+            .fetch_optional(db)
+            .await?
+            .flatten()
+            .map(|x| serde_json::from_value(x).ok())
+            .flatten()
+            .unwrap_or_default();
+
+    let config_smtp = if let (Some(host), Some(username), Some(password)) =
+        (config.smtp_host, config.smtp_username, config.smtp_password)
+    {
+        Some(Smtp {
+            host,
+            username,
+            password,
+            tls_implicit: config.smtp_tls_implicit.unwrap_or(false),
+            port: config.smtp_port.unwrap_or(587),
+            from: config
+                .smtp_from
+                .unwrap_or_else(|| "noreply@getwindmill.com".to_string()),
+        })
+    } else {
+        None
+    };
+    let smtp = config_smtp.or(
+        if let (Some(host), Some(username), Some(password)) = (
+            std::env::var("SMTP_HOST").ok(),
+            std::env::var("SMTP_USERNAME").ok(),
+            std::env::var("SMTP_PASSWORD").ok(),
+        ) {
+            Some(Smtp {
+                host,
+                username,
+                password,
+                tls_implicit: std::env::var("SMTP_TLS_IMPLICIT")
+                    .ok()
+                    .and_then(|p| p.parse().ok())
+                    .unwrap_or(false),
+                port: std::env::var("SMTP_PORT")
+                    .ok()
+                    .and_then(|p| p.parse().ok())
+                    .unwrap_or(587),
+                from: std::env::var("SMTP_FROM")
+                    .unwrap_or_else(|_| "noreply@getwindmill.com".to_string()),
+            })
+        } else {
+            None
+        },
+    );
+    if smtp.is_none() {
+        tracing::warn!("SMTP not configured");
+    }
+
+    Ok(ServerConfig {
+        smtp,
+        timeout_wait_result: config
+            .timeout_wait_result
+            .ok_or(
+                std::env::var("TIMEOUT_WAIT_RESULT")
+                    .ok()
+                    .and_then(|x| x.parse::<u64>().ok()),
+            )
+            .unwrap_or(600),
+    })
+}
+
+impl Default for ServerConfigOpt {
+    fn default() -> Self {
+        Self {
+            smtp_from: None,
+            smtp_host: None,
+            smtp_password: None,
+            smtp_port: None,
+            smtp_tls_implicit: None,
+            smtp_username: None,
+            timeout_wait_result: Default::default(),
+        }
+    }
+}
+
+#[derive(PartialEq, Clone, Debug)]
+pub struct ServerConfig {
+    pub smtp: Option<Smtp>,
+    pub timeout_wait_result: u64,
+}

backend/windmill-common/src/users.rs

@@ -15,3 +15,9 @@ pub fn username_to_permissioned_as(user: &str) -> String {
         format!("u/{}", user)
     }
 }
+
+pub fn truncate_token(token: &str) -> String {
+    let mut s = token[..10].to_owned();
+    s.push_str("*****");
+    s
+}

backend/windmill-common/src/utils.rs

@@ -79,7 +79,32 @@ pub async fn list_elems_from_hub(
     url: &str,
     email: &str,
 ) -> Result<serde_json::Value> {
-    let rows = http_get_from_hub(http_client, url, email, false)
+    let rows = http_get_from_hub(http_client, url, email, false, None)
+        .await?
+        .json::<serde_json::Value>()
+        .await
+        .map_err(crate::error::to_anyhow)?;
+    Ok(rows)
+}
+
+#[cfg(feature = "reqwest")]
+pub async fn query_elems_from_hub(
+    http_client: &reqwest::Client,
+    url: &str,
+    email: &str,
+    query_text: &str,
+    query_kind: &Option<String>,
+    query_limit: &Option<i64>,
+) -> Result<serde_json::Value> {
+    let mut query_params = vec![("text", query_text)];
+    if let Some(query_kind) = query_kind {
+        query_params.push(("kind", query_kind.as_str()));
+    }
+    let query_limit = query_limit.unwrap_or(0).to_string();
+    if query_limit.parse::<i64>().unwrap() > 0 {
+        query_params.push(("limit", query_limit.as_str()));
+    }
+    let rows = http_get_from_hub(http_client, url, email, false, Some(query_params))
         .await?
         .json::<serde_json::Value>()
         .await
@@ -93,8 +118,9 @@ pub async fn http_get_from_hub(
     url: &str,
     email: &str,
     plain: bool,
+    query_params: Option<Vec<(&str, &str)>>,
 ) -> Result<reqwest::Response> {
-    let response = http_client
+    let mut request = http_client
         .get(url)
         .header(
             "Accept",
@@ -104,10 +130,15 @@ pub async fn http_get_from_hub(
                 "application/json"
             },
         )
-        .header("X-email", email)
-        .send()
-        .await
-        .map_err(crate::error::to_anyhow)?;
+        .header("X-email", email);
+
+    if let Some(query_params) = query_params {
+        for (key, value) in query_params {
+            request = request.query(&[(key, value)]);
+        }
+    }
+
+    let response = request.send().await.map_err(crate::error::to_anyhow)?;
 
     Ok(response)
 }

backend/windmill-common/src/variables.rs

@@ -61,7 +61,7 @@ pub struct CreateVariable {
     pub is_oauth: Option<bool>,
 }
 
-pub fn get_reserved_variables(
+pub async fn get_reserved_variables(
     w_id: &str,
     token: &str,
     email: &str,
@@ -163,7 +163,7 @@ pub fn get_reserved_variables(
         },
         ContextualVariable {
             name: "WM_BASE_URL".to_string(),
-            value: BASE_URL.clone(),
+            value: BASE_URL.read().await.clone(),
             description: "base url of this instance".to_string(),
         },
         ContextualVariable {

backend/windmill-common/src/worker.rs

@@ -0,0 +1,206 @@
+use std::{collections::HashMap, sync::Arc};
+
+use itertools::Itertools;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use tokio::sync::RwLock;
+
+use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
+
+lazy_static::lazy_static! {
+    pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
+
+    pub static ref DEFAULT_TAGS : Vec<String> = vec![
+        "deno".to_string(),
+        "python3".to_string(),
+        "go".to_string(),
+        "bash".to_string(),
+        "powershell".to_string(),
+        "nativets".to_string(),
+        "mysql".to_string(),
+        "graphql".to_string(),
+        "bun".to_string(),
+        "postgresql".to_string(),
+        "bigquery".to_string(),
+        "snowflake".to_string(),
+        "graphql".to_string(),
+        "dependency".to_string(),
+        "flow".to_string(),
+        "hub".to_string(),
+        "other".to_string()];
+
+
+    pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
+        worker_tags: Default::default(),
+        dedicated_worker: Default::default(),
+    }));
+
+    pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
+
+
+
+    pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
+
+    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
+
+
+    pub static ref CUSTOM_TAGS_PER_WORKSPACE: Arc<RwLock<(Vec<String>, HashMap<String, Vec<String>>)>> = Arc::new(RwLock::new((vec![], HashMap::new())));
+
+    pub static ref ALL_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(vec![]));
+
+    static ref CUSTOM_TAG_REGEX: Regex =  Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
+
+}
+
+pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        CUSTOM_TAGS_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let tags = if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<Vec<String>>(q.value.clone()) {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse custom tags setting as vec of strings, found: {:#?}",
+                &q.value
+            );
+            vec![]
+        }
+    } else {
+        CUSTOM_TAGS.clone()
+    };
+
+    let custom_tags = process_custom_tags(tags);
+
+    tracing::info!(
+        "Loaded setting custom tags, common: {:?}, per-workspace: {:?}",
+        custom_tags.0,
+        custom_tags.1,
+    );
+
+    {
+        let mut l = CUSTOM_TAGS_PER_WORKSPACE.write().await;
+        *l = custom_tags.clone()
+    }
+    {
+        let mut l = ALL_TAGS.write().await;
+        *l = [
+            custom_tags.0.clone(),
+            custom_tags.1.keys().map(|x| x.to_string()).collect_vec(),
+        ]
+        .concat();
+    }
+    Ok(())
+}
+
+fn process_custom_tags(tags: Vec<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
+    let mut global = vec![];
+    let mut specific: HashMap<String, Vec<String>> = HashMap::new();
+    for e in tags {
+        if let Some(cap) = CUSTOM_TAG_REGEX.captures(&e) {
+            let tag = cap.get(1).unwrap().as_str().to_string();
+            let workspaces = cap.get(2).unwrap().as_str().split("+");
+            specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
+        } else {
+            global.push(e.to_string());
+        }
+    }
+    (global, specific)
+}
+
+pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
+    let (tags, dw) = {
+        let wc = WORKER_CONFIG.read().await.clone();
+        (
+            wc.worker_tags,
+            wc.dedicated_worker
+                .as_ref()
+                .map(|x| format!("{}:{}", x.workspace_id, x.path)),
+        )
+    };
+    sqlx::query!(
+        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+        worker_instance,
+        worker_name,
+        ip,
+        tags.as_slice(),
+        *WORKER_GROUP,
+        dw
+    )
+    .execute(db)
+    .await
+    .expect("insert worker_ping initial value");
+}
+
+pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
+    let mut config: WorkerConfigOpt = sqlx::query_scalar!(
+        "SELECT config FROM config WHERE name = $1",
+        format!("worker__{}", *WORKER_GROUP)
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten()
+    .map(|x| serde_json::from_value(x).ok())
+    .flatten()
+    .unwrap_or_default();
+    if config.dedicated_worker.is_none() {
+        config.dedicated_worker = std::env::var("DEDICATED_WORKER").ok();
+    }
+    let dedicated_worker = config.dedicated_worker.map(|x| {
+        let splitted = x.split(':').to_owned().collect_vec();
+        if splitted.len() != 2 {
+            panic!("DEDICATED_WORKER setting should be in the form of <workspace>:<script_path>")
+        } else {
+            let workspace = splitted[0];
+            let script_path = splitted[1];
+            WorkspacedPath { workspace_id: workspace.to_string(), path: script_path.to_string() }
+        }
+    });
+    Ok(WorkerConfig {
+        worker_tags: config
+            .worker_tags
+            .or_else(|| {
+                if let Some(ref dedicated_worker) = dedicated_worker.as_ref() {
+                    Some(vec![format!(
+                        "{}:{}",
+                        dedicated_worker.workspace_id, dedicated_worker.path
+                    )])
+                } else {
+                    std::env::var("WORKER_TAGS")
+                        .ok()
+                        .map(|x| x.split(',').map(|x| x.to_string()).collect())
+                }
+            })
+            .unwrap_or_else(|| DEFAULT_TAGS.clone()),
+        dedicated_worker,
+    })
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct WorkspacedPath {
+    pub workspace_id: String,
+    pub path: String,
+}
+#[derive(Serialize, Deserialize)]
+pub struct WorkerConfigOpt {
+    pub worker_tags: Option<Vec<String>>,
+    pub dedicated_worker: Option<String>,
+}
+
+impl Default for WorkerConfigOpt {
+    fn default() -> Self {
+        Self { worker_tags: Default::default(), dedicated_worker: Default::default() }
+    }
+}
+
+#[derive(PartialEq, Debug, Clone)]
+pub struct WorkerConfig {
+    pub worker_tags: Vec<String>,
+    pub dedicated_worker: Option<WorkspacedPath>,
+}

backend/windmill-queue/Cargo.toml

@@ -37,3 +37,4 @@ tokio.workspace = true
 futures-core.workspace = true
 itertools.workspace = true
 async-recursion.workspace = true
+bigdecimal.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -6,19 +6,18 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{collections::HashMap, sync::atomic::AtomicBool, vec};
-
-#[cfg(feature = "benchmark")]
-use std::time::Instant;
+use std::{collections::HashMap, vec};
 
 use anyhow::Context;
 use async_recursion::async_recursion;
+use bigdecimal::ToPrimitive;
 use chrono::{DateTime, Duration, Utc};
-use itertools::Itertools;
 use reqwest::Client;
 use rsmq_async::RsmqConnection;
 use serde_json::json;
 use sqlx::{Pool, Postgres, Transaction};
+#[cfg(feature = "benchmark")]
+use std::time::Instant;
 use tracing::{instrument, Instrument};
 use ulid::Ulid;
 use uuid::Uuid;
@@ -37,9 +36,13 @@ use windmill_common::{
     schedule::{schedule_to_user, Schedule},
     scripts::{ScriptHash, ScriptLang},
     users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL},
+    worker::WORKER_CONFIG,
     DB, METRICS_ENABLED,
 };
 
+#[cfg(feature = "enterprise")]
+use windmill_common::worker::CLOUD_HOSTED;
+
 use crate::{
     schedule::{get_schedule_opt, push_scheduled_job},
     QueueTransaction,
@@ -66,62 +69,7 @@ lazy_static::lazy_static! {
         "Total number of jobs pulled from the queue."
     )
     .unwrap();
-    pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
 
-    pub static ref DEFAULT_TAGS : Vec<String> = vec![
-        "deno".to_string(),
-        "python3".to_string(),
-        "go".to_string(),
-        "bash".to_string(),
-        "powershell".to_string(),
-        "nativets".to_string(),
-        "mysql".to_string(),
-        "graphql".to_string(),
-        "bun".to_string(),
-        "postgresql".to_string(),
-        "bigquery".to_string(),
-        "snowflake".to_string(),
-        "graphql".to_string(),
-        "dependency".to_string(),
-        "flow".to_string(),
-        "hub".to_string(),
-        "other".to_string()];
-
-    pub static ref DEDICATED_WORKER: Option<(String, String)> = std::env::var("DEDICATED_WORKER")
-        .ok()
-        .map(|x| {
-            let splitted = x.split(':').to_owned().collect_vec();
-            if splitted.len() != 2 {
-                panic!("DEDICATED_WORKER should be in the form of <workspace>:<script_path>")
-            } else {
-                let workspace = splitted[0];
-                let script_path = splitted[1];
-                (workspace.to_string(), script_path.to_string())
-            }
-        });
-
-    pub static ref ACCEPTED_TAGS: Vec<String> = {
-        let worker_tags = std::env::var("WORKER_TAGS")
-        .ok()
-        .map(|x| x.split(',').map(|x| x.to_string()).collect())
-        .unwrap_or_else(|| DEFAULT_TAGS.clone());
-        if let Some(ref dedicated_worker) = DEDICATED_WORKER.as_ref() {
-            vec![format!("{}:{}", dedicated_worker.0, dedicated_worker.1)]
-        } else {
-            worker_tags
-         }
-    };
-
-    pub static ref IS_WORKER_TAGS_DEFINED: bool = std::env::var("WORKER_TAGS").ok().is_some();
-
-
-
-
-
-    // When compiled in 'benchmark' mode, this flags is exposed via the /workers/toggle endpoint
-    // and make it possible to disable to current active workers (such that they don't pull any)
-    // jobs from the queue
-    pub static ref IDLE_WORKERS: AtomicBool = AtomicBool::new(false);
 }
 
 #[cfg(feature = "enterprise")]
@@ -284,6 +232,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
             .await
             .ok()
             .flatten()
+            .map(|x| x.to_i64())
+            .flatten()
         } else {
             tracing::warn!("Could not parse flow status");
             None
@@ -300,7 +250,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         .flatten();
     let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
     let job_id = queued_job.id.clone();
-    let _duration = sqlx::query_scalar!(
+    let _duration: i64 = sqlx::query_scalar!(
         "INSERT INTO completed_job AS cj
                    ( workspace_id
                    , id
@@ -425,7 +375,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     if !is_flow && _duration > 1000 {
         let additional_usage = _duration / 1000;
         let w_id = &queued_job.workspace_id;
-        let premium_workspace = *CLOUD_HOSTED
+        let premium_workspace = *windmill_common::worker::CLOUD_HOSTED
             && sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
                 .fetch_one(db)
                 .await
@@ -436,7 +386,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
                 ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $3",
                 if premium_workspace { w_id } else { &queued_job.email },
                 premium_workspace,
-                additional_usage)
+                additional_usage as i32)
                 .execute(db)
                 .await
                 .map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
@@ -1072,8 +1022,8 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
 
         let job_uuid: Uuid = pulled_job.id;
         let min_started_at: Option<DateTime<Utc>> = script_path_live_stats.min_started_at;
-        let avg_script_duration: Option<i32> = sqlx::query_scalar!(
-            "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM
+        let avg_script_duration: Option<i64> = sqlx::query_scalar!(
+            "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM
                 (SELECT duration_ms FROM completed_job WHERE script_path = $1
                 ORDER BY started_at
                 DESC LIMIT 10) AS t",
@@ -1127,7 +1077,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
                 , started_at = null
                 , scheduled_for = '{estimated_next_schedule_timestamp}'
                 , logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END
-                WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false)
+                WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())
                 RETURNING *"
             ))
             .fetch_all(&mut tx)
@@ -1153,7 +1103,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
         // TODO: REDIS: Race conditions / replace last_ping
 
         // TODO: shuffle this list to have fairness
-        let mut all_tags = ACCEPTED_TAGS.clone();
+        let mut all_tags = WORKER_CONFIG.read().await.worker_tags.clone();
 
         let mut msg: Option<_> = None;
         let mut tag = None;
@@ -1210,7 +1160,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
          *   suspend_until is non-null
          *   and suspend = 0 when the resume messages are received
          *   or suspend_until <= now() if it has timed out */
-
+        let config = WORKER_CONFIG.read().await.clone();
+        let tags = config.worker_tags.clone();
+        drop(config);
         let r = if suspend_first {
             sqlx::query_as::<_, QueuedJob>("UPDATE queue
             SET running = true
@@ -1226,7 +1178,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
                 LIMIT 1
             )
             RETURNING *")
-                .bind(ACCEPTED_TAGS.as_slice())
+                .bind(tags)
                 .fetch_optional(db)
                 .await?
         } else {
@@ -1237,6 +1189,8 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
             // #[cfg(feature = "benchmark")]
             // let instant = Instant::now();
 
+            let tags = WORKER_CONFIG.read().await.worker_tags.clone();
+
             let r = sqlx::query_as::<_, QueuedJob>(
                 "UPDATE queue
             SET running = true
@@ -1253,10 +1207,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
             )
             RETURNING *",
             )
-            .bind(ACCEPTED_TAGS.as_slice())
+            .bind(tags)
             .fetch_optional(db)
             .await?;
-
             // #[cfg(feature = "benchmark")]
             // println!("pull query: {:?}", instant.elapsed());
 
@@ -1353,15 +1306,35 @@ pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
         w_id,
         job_id
     )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
-    .unwrap_or(0)
-        == 1;
-    tracing::debug!("Job {job_id} deleted: {job_removed}");
+    .fetch_optional(&mut tx)
+    .await;
+
+    if let Err(job_removed) = job_removed {
+        tracing::error!(
+            "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"
+        );
+    } else {
+        let job_removed = job_removed.unwrap().flatten().unwrap_or(0);
+        if job_removed != 1 {
+            tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling");
+        }
+    }
+
+    tracing::debug!("Job {job_id} deleted");
     Ok(tx)
 }
 
+pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result<bool> {
+    Ok(sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
+        id,
+        w_id
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(false))
+}
+
 pub async fn get_queued_job<'c>(
     id: Uuid,
     w_id: &str,
@@ -1771,11 +1744,11 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             tag = None;
         }
         let default = || {
-            if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
+            if job_kind == JobKind::Flow
+                || job_kind == JobKind::FlowPreview
+                || job_kind == JobKind::Identity
+            {
                 "flow".to_string()
-            } else if job_kind == JobKind::Identity {
-                // identity is a light script, nativets is too
-                "nativets".to_string()
             } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
                 "dependency".to_string()
             } else {

backend/windmill-worker/Cargo.toml

@@ -71,6 +71,7 @@ jsonwebtoken = { workspace = true, optional = true }
 sha2 = { workspace = true, optional = true }
 pem = { workspace = true, optional = true }
 urlencoding.workspace = true
+nix.workspace = true
 
 [build-dependencies]
 deno_fetch.workspace = true

backend/windmill-worker/loader.bun.ts

@@ -24,15 +24,25 @@ const p = {
         loader: "tsx",
       };
     });
-    build.onResolve({ filter: /(?!\.\/main\.ts)\..*\.ts$/ }, (args) => {
-      const cdir = resolve("./");
+    const cdir = resolve("./");
+    const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos
+    const filter = new RegExp(
+      `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.ts).*\\.ts$`
+    );
+    build.onResolve({ filter }, (args) => {
       const file_path =
         args.importer == "./main.ts" || args.importer == resolve("./main.ts")
           ? current_path
           : args.importer.replace(cdir + "/", "");
 
-      const url = `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`;
-      const file = resolve("./" + current_path + "/../" + args.path + ".url");
+      const isRelative = !args.path.startsWith("/");
+
+      const url = isRelative
+        ? `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`
+        : `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${args.path}`;
+      const file = isRelative
+        ? resolve("./" + current_path + "/../" + args.path + ".url")
+        : resolve("./" + args.path + ".url");
       mkdirSync(dirname(file), { recursive: true });
       writeFileSync(file, url);
       return {

backend/windmill-worker/loader_builder.bun.ts

@@ -7,6 +7,8 @@ const bo = await Bun.build({
 
 const fs = require("fs/promises");
 
+const captureVersion = /(^\@?[^\@]+)(?:\@(.+))?/;
+
 if (!bo.success) {
   bo.logs.forEach((l) => console.log(l));
   process.exit(1);
@@ -17,7 +19,7 @@ if (!bo.success) {
   const { intersect } = require("semver-intersect");
   const dependencies: Record<string, string[]> = {};
   for (const i of imports) {
-    let [name, version] = i.path.split("@");
+    let [_, name, version] = i.path.match(captureVersion) ?? [];
     if (version == undefined) {
       if (dependencies[name] == undefined) {
         dependencies[name] = [];