all

* cache & inits

* cache & inits

* cache & inits

* all

* fix tests

* improve

.env

@@ -1,19 +1,12 @@
-DB_PASSWORD=changeme
 DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
 
 WM_IMAGE=ghcr.io/windmill-labs/windmill:main
 WM_LICENSE_KEY=""
 
 # For Enterprise Edition, comment the 2 lines above and uncomment below
-# WINDMILL_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
+# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
 # WM_LICENSE_KEY="<id>.<expiry>.<signature>"
 
 
-# this is the url that your instance is publicly exposed to
-WM_BASE_URL=http://localhost
-
 # To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
 # To have caddy take care of automatic TLS
-
-
-WM_REQUEST_SIZE_LIMIT=50097152 # 50MB (The size limit for any requests, including script/flow args inputs)

.github/DockerfileBackendTests

@@ -37,9 +37,10 @@ RUN apt-get update \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr nodejs npm\
+    libv8-dev tesseract-ocr nodejs npm clang libclang-dev\
     && rm -rf /var/lib/apt/lists/*
 
+
 RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
@@ -57,7 +58,7 @@ RUN /usr/local/bin/python3 -m pip install nltk
 RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
+COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
 RUN apt-get update \

.github/change-versions.sh

@@ -5,7 +5,7 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json

.github/workflows/backend-test.yml

@@ -42,4 +42,4 @@ jobs:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
           DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
-          test --all -- --nocapture
+          test --features enterprise --all -- --nocapture 

.github/workflows/benchmark.yml

@@ -19,15 +19,28 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
       windmill:
-        image: ghcr.io/windmill-labs/windmill:main
+        image: ghcr.io/windmill-labs/windmill-ee:main
         env:
           DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow
         options: >-
           --pull always --health-interval 10s --health-timeout 5s
           --health-retries 5 --health-cmd "curl
           http://localhost:8000/api/version"
         ports:
           - 8000:8000
+      windmill-worker:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          DISABLE_SERVER: true
+          WORKER_GROUP: dedicated
+          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+        options: >-
+          --pull always
     steps:
       - uses: denoland/setup-deno@v1
         with:
@@ -37,13 +50,9 @@ jobs:
           ref: benchmarks
       - name: benchmark
         timeout-minutes: 10
-        run: deno run --unstable -A
+        run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          --host http://localhost:8000
-          -e admin@windmill.dev
-          -p changeme
           -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
-          --branch ${GITHUB_REF##ref/head/}
       - name: Push changes
         run: |
           pwd

CHANGELOG.md

@@ -1,6 +1,248 @@
 # Changelog
 
 
+## [1.181.0](https://github.com/windmill-labs/windmill/compare/v1.180.0...v1.181.0) (2023-10-05)
+
+
+### Features
+
+* add npm_config_registry support for bun, deno and being settable from UI ([#2373](https://github.com/windmill-labs/windmill/issues/2373)) ([c42b875](https://github.com/windmill-labs/windmill/commit/c42b8750f1d41c9b4de6c96f1ea82239c5325495))
+* **frontend:** add driverjs ([#2327](https://github.com/windmill-labs/windmill/issues/2327)) ([bda6f1f](https://github.com/windmill-labs/windmill/commit/bda6f1fe5d44a3c1d925c1b8a8e872d9f5fba484))
+
+
+### Bug Fixes
+
+* add numeric, array and date types ([#2379](https://github.com/windmill-labs/windmill/issues/2379)) ([768f972](https://github.com/windmill-labs/windmill/commit/768f972cbf578b3394f89120d172b02bcaac5413))
+* add reserved variables in args ([#2371](https://github.com/windmill-labs/windmill/issues/2371)) ([e7165f3](https://github.com/windmill-labs/windmill/commit/e7165f3357a2ba7a690accd78a03c2518aa61860))
+* ai flow prompt fix + explanation in ui ([#2374](https://github.com/windmill-labs/windmill/issues/2374)) ([66d15f0](https://github.com/windmill-labs/windmill/commit/66d15f0c17698077c5bf299af8368e9cfdbf3ecb))
+* flow trigger prompt + lower temp ([#2377](https://github.com/windmill-labs/windmill/issues/2377)) ([733bfe3](https://github.com/windmill-labs/windmill/commit/733bfe3f14e6eb0237c6a528ab64ae71082a4679))
+* **frontend:** fix flow tutorials ([#2383](https://github.com/windmill-labs/windmill/issues/2383)) ([63ad53f](https://github.com/windmill-labs/windmill/commit/63ad53fa70c4f1769d873bff962bfb2d66081163))
+* schema autocomplete/ai ([#2372](https://github.com/windmill-labs/windmill/issues/2372)) ([9ed748a](https://github.com/windmill-labs/windmill/commit/9ed748a0dac95f152f91de6e25b63d841af0dd50))
+* trigger bun prompt ([#2368](https://github.com/windmill-labs/windmill/issues/2368)) ([fc9adbe](https://github.com/windmill-labs/windmill/commit/fc9adbe56081065fa3de662e664fcebe0f4c25ee))
+
+## [1.180.0](https://github.com/windmill-labs/windmill/compare/v1.179.1...v1.180.0) (2023-10-01)
+
+
+### Features
+
+* code content search ([#2367](https://github.com/windmill-labs/windmill/issues/2367)) ([fb96059](https://github.com/windmill-labs/windmill/commit/fb960594fce265d5d4f4eb443e0c9cc19d14e025))
+
+
+### Bug Fixes
+
+* improve connection in apps ([a2fca17](https://github.com/windmill-labs/windmill/commit/a2fca17ae2ac8257154e2aec4a0ceabfe16fc46a))
+
+## [1.179.1](https://github.com/windmill-labs/windmill/compare/v1.179.0...v1.179.1) (2023-09-30)
+
+
+### Bug Fixes
+
+* fix 0 len flow module processing ([f97289a](https://github.com/windmill-labs/windmill/commit/f97289a3d8bc6ce978d0be1fec35a424211e4a20))
+
+## [1.179.0](https://github.com/windmill-labs/windmill/compare/v1.178.1...v1.179.0) (2023-09-30)
+
+
+### Features
+
+* add trustedDependencies escape hatch for bun ([#2364](https://github.com/windmill-labs/windmill/issues/2364)) ([52df265](https://github.com/windmill-labs/windmill/commit/52df2650ea5d5c03e94c96af0b8a79275856fc37))
+* ai code completion ([#2361](https://github.com/windmill-labs/windmill/issues/2361)) ([0937706](https://github.com/windmill-labs/windmill/commit/093770692ac40b8ee0139f24d63bcccda9bf6ddb))
+* **backend:** parse expires_in from string in TokenResponse ([#2353](https://github.com/windmill-labs/windmill/issues/2353)) ([4621915](https://github.com/windmill-labs/windmill/commit/46219154de07ef5a6e071f1c2859cea35c7f9943))
+* **frontend:** copy schema from json and past runs in flow inputs ([#2352](https://github.com/windmill-labs/windmill/issues/2352)) ([3cb2977](https://github.com/windmill-labs/windmill/commit/3cb29778dd70199d9504aa7c1a12bfd7a02569d6))
+
+
+### Bug Fixes
+
+* error handler does not recover flow anymore and error handler is called only once up the flow ([445bf96](https://github.com/windmill-labs/windmill/commit/445bf965eddc6da39a125fce60b53e0903698664))
+* **frontend:** Properly handle click ([#2351](https://github.com/windmill-labs/windmill/issues/2351)) ([55b7f98](https://github.com/windmill-labs/windmill/commit/55b7f982c2bbbb5d4daa9752ec8ffc0c79c374fc))
+* **frontend:** timezone fix ([#2360](https://github.com/windmill-labs/windmill/issues/2360)) ([dcfa5fc](https://github.com/windmill-labs/windmill/commit/dcfa5fc0e40f5cd8dba5a26be31695ce765c7e23))
+* improve superadmin settings page ([b029027](https://github.com/windmill-labs/windmill/commit/b029027c1c75c0b6489966371db7d2f9c99d15f8))
+* non skipped failures stop even in presence of an error handler ([1c5cc0c](https://github.com/windmill-labs/windmill/commit/1c5cc0c237101caf6c5e6e34b11c967a27cd4112))
+* remove shared http clients in rest runtime ([4931ed9](https://github.com/windmill-labs/windmill/commit/4931ed95c4b12f63effa1dd7d6a5cd526a612302))
+
+## [1.178.1](https://github.com/windmill-labs/windmill/compare/v1.178.0...v1.178.1) (2023-09-28)
+
+
+### Bug Fixes
+
+* improve license key check ([035bad5](https://github.com/windmill-labs/windmill/commit/035bad5268d182af3f30915b5356defd7f6ccbc0))
+
+## [1.178.0](https://github.com/windmill-labs/windmill/compare/v1.177.1...v1.178.0) (2023-09-28)
+
+
+### Features
+
+* **frontend:** add app groups management ([#2347](https://github.com/windmill-labs/windmill/issues/2347)) ([20e0427](https://github.com/windmill-labs/windmill/commit/20e0427a1303c1c32f41b198cd2d0f7f28b5bd32))
+* **frontend:** add AppDrawer controls ([#2339](https://github.com/windmill-labs/windmill/issues/2339)) ([3de6d44](https://github.com/windmill-labs/windmill/commit/3de6d446f281dcaac288deee19342a08e0ccf9af))
+* **frontend:** Switch to component list when deleting a component ([#2346](https://github.com/windmill-labs/windmill/issues/2346)) ([6fcd72c](https://github.com/windmill-labs/windmill/commit/6fcd72c79453dd9d60ca869cd9996cc0c25971fa))
+
+
+### Bug Fixes
+
+* add env tags to default worker group ([#2348](https://github.com/windmill-labs/windmill/issues/2348)) ([f5bed95](https://github.com/windmill-labs/windmill/commit/f5bed95ab15bc397f822b06816c43b4b13a84af3))
+
+## [1.177.1](https://github.com/windmill-labs/windmill/compare/v1.177.0...v1.177.1) (2023-09-26)
+
+
+### Bug Fixes
+
+* **frontend:** fix modal closing issues ([#2340](https://github.com/windmill-labs/windmill/issues/2340)) ([18cf8fa](https://github.com/windmill-labs/windmill/commit/18cf8faec16d496e4b327505b682459ed518a5b4))
+* **frontend:** fix overflow ([#2341](https://github.com/windmill-labs/windmill/issues/2341)) ([2e8f2ec](https://github.com/windmill-labs/windmill/commit/2e8f2ec724f6802170121f4f8aa73b697a39c9ee))
+* improve list component handling of non array data ([dc44b08](https://github.com/windmill-labs/windmill/commit/dc44b0841af17227160b9d56ec446e6646a8ab0d))
+
+## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
+
+
+### Features
+
+* add custom oauth support ([#2336](https://github.com/windmill-labs/windmill/issues/2336)) ([01277f4](https://github.com/windmill-labs/windmill/commit/01277f4d3b8bb04b955d5bbb2ed69c1c7c8f4f9e))
+* support automatic reconnection to pg ([ccaa05d](https://github.com/windmill-labs/windmill/commit/ccaa05d4bf5954c3fb8678239d2962cac6550a5a))
+
+
+### Bug Fixes
+
+* fix resource type picker object reinitialization ([f0f15c4](https://github.com/windmill-labs/windmill/commit/f0f15c47cb35cc1e3cfa13549465803a1e970770))
+* **frontend:** Fix build ([#2330](https://github.com/windmill-labs/windmill/issues/2330)) ([46592af](https://github.com/windmill-labs/windmill/commit/46592affd3d51b54632a2a7a281c11141edcb4a5))
+* **frontend:** Fix markdown dark mode ([#2329](https://github.com/windmill-labs/windmill/issues/2329)) ([6c19740](https://github.com/windmill-labs/windmill/commit/6c197407185810f43c47d4107007bd69814a1d65))
+* set min size of components to 1 ([d298093](https://github.com/windmill-labs/windmill/commit/d298093e29bd9983c7631a8f8c80e47b768bb93c))
+
+## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24)
+
+
+### Features
+
+* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254))
+* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6))
+* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33))
+* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85))
+* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8))
+
+
+### Bug Fixes
+
+* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0))
+* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64))
+* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1))
+* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9))
+* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781))
+* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55))
+* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9))
+* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91))
+
+## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19)
+
+
+### Features
+
+* add batch jobs ([#2306](https://github.com/windmill-labs/windmill/issues/2306)) ([5867e5d](https://github.com/windmill-labs/windmill/commit/5867e5d0f80fd515fab165659831b5ee9a8c3f97))
+* add dediacted worker env var ([#2296](https://github.com/windmill-labs/windmill/issues/2296)) ([e0c6eee](https://github.com/windmill-labs/windmill/commit/e0c6eee16e535b3a7d803a7978e463404f5fec30))
+* dedicated benchmarks ([#2297](https://github.com/windmill-labs/windmill/issues/2297)) ([c549239](https://github.com/windmill-labs/windmill/commit/c5492396843ddd9143ffe890696d0317c970de36))
+* **frontend:** Add component control doc ([#2295](https://github.com/windmill-labs/windmill/issues/2295)) ([26f8863](https://github.com/windmill-labs/windmill/commit/26f88636f0b972d4fe4931ed02135c38b27a56d2))
+* suggest adding openai key on workspace creation ([a6b3b2f](https://github.com/windmill-labs/windmill/commit/a6b3b2f63b317825a3d80218cbb606b9f610c221))
+* support pinned versions for bun in deployed scripts ([03806dc](https://github.com/windmill-labs/windmill/commit/03806dc3907cba724be14acb6aadf5be6e35cdb6))
+
+
+### Bug Fixes
+
+* add HOME to bun and deno ([0e3ecc7](https://github.com/windmill-labs/windmill/commit/0e3ecc7d6025c173135f20bacc33a0dc972ec222))
+* add queue_count to metrics ([9ced883](https://github.com/windmill-labs/windmill/commit/9ced8834a45151c6900b1eb33eca2cff4886a065))
+* ai improve prompts ([#2310](https://github.com/windmill-labs/windmill/issues/2310)) ([b647213](https://github.com/windmill-labs/windmill/commit/b647213b2c968b0cb1f90c97d94e8023c415dd55))
+* **frontend:** add missing key ([#2299](https://github.com/windmill-labs/windmill/issues/2299)) ([39d2467](https://github.com/windmill-labs/windmill/commit/39d24672ddd696372e55e9b4566f322a322385a8))
+* **frontend:** Always mount components ([#2309](https://github.com/windmill-labs/windmill/issues/2309)) ([34f94aa](https://github.com/windmill-labs/windmill/commit/34f94aa50e92254114c046fa8b7e900d93807937))
+* **frontend:** fix alignment ([#2307](https://github.com/windmill-labs/windmill/issues/2307)) ([f9fc6f1](https://github.com/windmill-labs/windmill/commit/f9fc6f19482e68c9ccba0014879fd8761662c36a))
+* **frontend:** Fix rich result styling + add title and hideDetails config ([#2294](https://github.com/windmill-labs/windmill/issues/2294)) ([732daef](https://github.com/windmill-labs/windmill/commit/732daef1c3515f7df3e09deac691bb585f9859cd))
+* **frontend:** fix tab styling + component bg ([#2308](https://github.com/windmill-labs/windmill/issues/2308)) ([5e773d3](https://github.com/windmill-labs/windmill/commit/5e773d386343f003425173207c166e3c4eeef956))
+* **frontend:** fix theme make default ([#2304](https://github.com/windmill-labs/windmill/issues/2304)) ([4629819](https://github.com/windmill-labs/windmill/commit/46298197c5333a81b9b8a004027ab9a856bdada4))
+* **frontend:** fix theme UI ([#2305](https://github.com/windmill-labs/windmill/issues/2305)) ([576f76b](https://github.com/windmill-labs/windmill/commit/576f76b1ffe9c50c8ccaca8c5e34d0ec03aebf3f))
+* validate more strongly usernames ([47094bb](https://github.com/windmill-labs/windmill/commit/47094bb8d1c6f4ba621d42515dede061fd04afdd))
+
+## [1.174.0](https://github.com/windmill-labs/windmill/compare/v1.173.0...v1.174.0) (2023-09-15)
+
+
+### Features
+
+* ai gen support all langs ([#2276](https://github.com/windmill-labs/windmill/issues/2276)) ([39590b3](https://github.com/windmill-labs/windmill/commit/39590b3d2592b2d08117c0f70829c13f1efb4885))
+* bun absolute/relative imports + tests ([#2286](https://github.com/windmill-labs/windmill/issues/2286)) ([e5ce85b](https://github.com/windmill-labs/windmill/commit/e5ce85b9affe665342f24b1d39ce3d03db09b941))
+* **frontend:** Global CSS editor ([#2178](https://github.com/windmill-labs/windmill/issues/2178)) ([7e9ee39](https://github.com/windmill-labs/windmill/commit/7e9ee39aa69bc31766b5e4f4aab498c8f14067cd))
+
+## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
+
+
+### Features
+
+* cli sync on windows ([#2283](https://github.com/windmill-labs/windmill/issues/2283)) ([c371cb3](https://github.com/windmill-labs/windmill/commit/c371cb397ab3d0c534e2c553d1dfb1ad5176d2a6))
+
+
+### Bug Fixes
+
+* accept jobs whose duration > 24 days ([2c00894](https://github.com/windmill-labs/windmill/commit/2c00894122aa8caee59b20625935284de6902950))
+
+## [1.172.1](https://github.com/windmill-labs/windmill/compare/v1.172.0...v1.172.1) (2023-09-14)
+
+
+### Bug Fixes
+
+* improve splitpane + improve deleting conditional tab ([1629008](https://github.com/windmill-labs/windmill/commit/1629008eb2eb48ff9cc2cf6b3a351efcf682244d))
+* update to svelte 4 ([#2280](https://github.com/windmill-labs/windmill/issues/2280)) ([90c10d8](https://github.com/windmill-labs/windmill/commit/90c10d803b4c47a9e1ac5b9e49e2a614344299a9))
+
+## [1.172.0](https://github.com/windmill-labs/windmill/compare/v1.171.0...v1.172.0) (2023-09-13)
+
+
+### Features
+
+* improve ai flow ([#2270](https://github.com/windmill-labs/windmill/issues/2270)) ([b23417a](https://github.com/windmill-labs/windmill/commit/b23417ab5b9938bbdf9db6449102760ff8c80152))
+* worker groups admin panel ([#2277](https://github.com/windmill-labs/windmill/issues/2277)) ([070b162](https://github.com/windmill-labs/windmill/commit/070b16222bc666866284180b3878f4d4f27bfa85))
+
+
+### Bug Fixes
+
+* ai flow nits ([#2272](https://github.com/windmill-labs/windmill/issues/2272)) ([8f6f46d](https://github.com/windmill-labs/windmill/commit/8f6f46de199d58133b9faa77cdbcbcfd6cb962f7))
+
+## [1.171.0](https://github.com/windmill-labs/windmill/compare/v1.170.0...v1.171.0) (2023-09-12)
+
+
+### Features
+
+* attempt to SIGTERM before SIGKILL for bash ([f40bbba](https://github.com/windmill-labs/windmill/commit/f40bbba519a97cbb1ec142c335f038dbebcd4e7c))
+* zero copy result for job result ([#2263](https://github.com/windmill-labs/windmill/issues/2263)) ([22a7da5](https://github.com/windmill-labs/windmill/commit/22a7da58b1d20721892906cba2dee6fbeb1cc1fd))
+
+
+### Bug Fixes
+
+* 2257 TIME  convertion in pg_executor.rs ([#2267](https://github.com/windmill-labs/windmill/issues/2267)) ([3d71253](https://github.com/windmill-labs/windmill/commit/3d71253abdb0dff1670a796d07a53ecd0a98414e))
+* fix field duplicate in app background settings ([164cdaf](https://github.com/windmill-labs/windmill/commit/164cdaf09464646dee4e70a699222a454eb0d898))
+* improve bun lockfile resolution ([9103ec4](https://github.com/windmill-labs/windmill/commit/9103ec445db81395a5851202eecb87301d0b4987))
+* remove result and args from list completed and list queue jobs ([e7e63e1](https://github.com/windmill-labs/windmill/commit/e7e63e111a73e0986050a8fe7fdc18784ba902b0))
+
+## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
+
+
+### Features
+
+* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
+* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
+
+
+### Bug Fixes
+
+* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
+
+## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
+
+
+### Features
+
+* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
+* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
+* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
+* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
+* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
+
+
+### Bug Fixes
+
+* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
+* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
+
 ## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
 
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:buster-slim as nsjail
+FROM debian:bookworm-slim as nsjail
 
 WORKDIR /nsjail
 
@@ -6,29 +6,29 @@ ARG nsjail=""
 
 RUN  if [ "$nsjail" = "true" ]; then apt-get -y update \
     && apt-get install -y \
-    bison=2:3.3.* \
+    bison=2:3.8.* \
     flex=2.6.* \
-    g++=4:8.3.* \
-    gcc=4:8.3.* \
-    git=1:2.20.* \
-    libprotobuf-dev=3.6.* \
-    libnl-route-3-dev=3.4.* \
-    make=4.2.* \
-    pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*; fi
+    g++=4:12.2.* \
+    gcc=4:12.2.* \
+    git=1:2.39.* \
+    libprotobuf-dev=3.21.* \
+    libnl-route-3-dev=3.7.* \
+    make=4.3-4.1 \
+    pkg-config=1.8.* \
+    protobuf-compiler=3.21.*; fi
 
 
 RUN if [ "$nsjail" = "true" ]; then git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800; fi
 RUN if [ "$nsjail" = "true" ]; then make; else touch nsjail; fi
 
-FROM rust:slim-buster AS rust_base
+FROM rust:slim-bookworm AS rust_base
 
 RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl nodejs npm
+    curl nodejs
 
 RUN rustup component add rustfmt
 
@@ -87,7 +87,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM debian:buster-slim as downloader
+FROM debian:bookworm-slim as downloader
 
 ARG TARGETPLATFORM
 
@@ -101,7 +101,7 @@ RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denol
 
 RUN unzip deno.zip && rm deno.zip
 
-FROM python:3.11.4-slim-buster
+FROM python:3.11.4-slim-bookworm
 
 ARG TARGETPLATFORM
 
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.2 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

LICENSE

@@ -1,16 +1,22 @@
-
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
+./LICENSE-AGPL)
 
-Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
-Every file is under License AGPL unless otherwise specified 
-or belonging to one of the below cases:
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
+specified. Every file is under License AGPL unless otherwise specified or
+belonging to one of the below cases:
 
-The files under backend/ are AGPLv3 Licensed.
-The files under frontend/ are AGPLv3 Licensed.
-The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+The files under backend/ are AGPLv3 Licensed, except any snippets of code under
+the compile flag "enterprise". Those snippets and files are under a proprietary
+and commercial license. The files under frontend/ are AGPLv3 Licensed, except
+any snippets of code that require a positive license check to be activated.
+Those snippets and files are under a proprietary and commercial license. Private
+and public forks MUST not include any of the above proprietary and commercial
+code. Windmill Labs, Inc. provide tools to clean the codebase from those
+snippets upon demand. The files under python-client/ deno-client/ go-client/ are
+Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
 
-All third party components incorporated into the Windmill Software are licensed under the
-original license provided by the owner of the applicable component.
+All third party components incorporated into the Windmill Software are licensed
+under the original license provided by the owner of the applicable component.

README.md

@@ -211,7 +211,6 @@ compiling from source or using without a postgres super user, see
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/oauth.json -o oauth.json
 
 docker compose up -d
 ```
@@ -227,11 +226,9 @@ From there, you can follow the setup app and create other users.
 We publish helm charts at:
 <https://github.com/windmill-labs/windmill-helm-charts>.
 
-### Postgres without superuser
+### OAuth, SSO & SMTP
 
-If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
-postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for Windmill.
+Windmill Community Edition allows to configure the OAuth, SSO (including Google Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin settings. Do note that there is a limit of 50 SSO users on the community edition.
 
 ### Commercial license
 
@@ -250,76 +247,6 @@ your current infrastructure to Windmill, support with tight SLA, and our global
 cache sync for high-performance/no dependency cache miss of cluster from 10+
 nodes to 200+ nodes.
 
-### OAuth for self-hosting
-
-To get the same oauth integrations as Windmill Cloud, mount `oauth.json` with
-the following format:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    "allowed_domains": ["windmill.dev"] //restrict a client OAuth login to some domains
-  }
-}
-```
-
-and mount it at `/usr/src/app/oauth.json`.
-
-The redirect url for the oauth clients is:
-`<instance_url>/user/login_callback/<client>`
-
-Even if you setup oauth, you will still want to **login as admin@windmill.dev /
-changeme** to setup your instance as a super-admin and give yourself admin
-rights.
-
-[The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
-
-To add more "connect an app" OAuth clients to the Windmill project, read the
-[Contributor's guide](https://www.windmill.dev/docs/misc/contributing). We
-welcome contributions!
-
-You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
-
-```json
-{
-  "<client>": {
-    "id": "<CLIENT_ID>",
-    "secret": "<CLIENT_SECRET>",
-    // To add a new OAuth2 IdP
-    "login_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "userinfo_url": "<userinfo endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    },
-    // To add a new OAuth2 Resource
-    "connect_config": {
-      "auth_url": "<auth_endpoint>",
-      "token_url": "<token_endpoint>",
-      "scopes": ["scope1", "scope2"],
-      "extra_params": "<if_needed>"
-    }
-  }
-}
-```
-
-### smtp for self-hosting
-
-For users to receive emails when you invite them to workspaces or add them to
-the instances using their emails, configure the SMTP env variables in the
-servers:
-
-```
-SMTP_FROM=noreply@windmill.dev
-SMTP_HOST=smtp.gmail.com
-SMTP_PORT=587
-SMTP_USERNAME=ruben@windmill.dev
-SMTP_PASSWORD=yourpasswordapp
-```
-
 ### Resource types
 
 You will also want to import all the approved resource types from
@@ -331,10 +258,9 @@ it being synced automatically everyday.
 | Environment Variable name                     | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
 | --------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
 | DATABASE_URL                                  |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| WORKER_GROUP                                  | default                                    | The worker group the worker belongs to and get its configuration pulled from                                                                                                                       | Worker                |
 | SERVER_BIND_ADDR                              | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
 | PORT                                          | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |
 | METRICS_ADDR                                  | None                                       | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                      | All                   |
 | JSON_FMT                                      | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
@@ -374,9 +300,11 @@ it being synced automatically everyday.
 | PIP_LOCAL_DEPENDENCIES                        | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
 | ADDITIONAL_PYTHON_PATHS                       | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
 | INCLUDE_HEADERS                               | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path>                                                                                                            | Worker                |
 | CUSTOM_TAGS                                   | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
 | JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
 | WAIT_RESULT_FAST_POLL_INTERVAL_MS             | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
@@ -395,12 +323,13 @@ it being synced automatically everyday.
 | SMTP_USERNAME                                 | None                                       | username for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
-| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | false                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
+| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                       | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
 | GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |
 | SECRET_SALT                                   | None                                       | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server    | Server + Worker       |
 | OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI API base path (no trailing slash)                                                                                                                                                     | Server                |
+| DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
 
 ## Run a local dev setup
 

backend/.sqlx/query-025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM global_settings WHERE name = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "025e5bf6aef56e9832239489daf21b4970d84529cf7ed7928ce2dcfec8b007f0"
+}

backend/.sqlx/query-034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = $1 WHERE workspace_id = $2",
+  "query": "UPDATE workspace_settings SET openai_resource_path = $1, code_completion_enabled = $2 WHERE workspace_id = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Varchar",
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "d7d368f9ef2587c6abd3daaec93401615a53e3d7d949b65ebf01ef8760ba46b8"
+  "hash": "034583442e6f8ae38d6c4e4aac26f17c8d9d0e657f28276228fc90d3e22e1304"
 }

backend/.sqlx/query-03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
+}

backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM config WHERE name = $1 RETURNING name",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a"
+}

backend/.sqlx/query-16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT script_path FROM completed_job  WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "script_path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd"
+}

backend/.sqlx/query-16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "16e4b1bead9fc77fd98658b8cb8cc6d6bf1df758b30e99bd661da866062ef14f"
+}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597"

backend/.sqlx/query-240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -37,6 +37,11 @@
         "ordinal": 6,
         "name": "custom_tags",
         "type_info": "TextArray"
+      },
+      {
+        "ordinal": 7,
+        "name": "worker_group",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -52,8 +57,9 @@
       false,
       false,
       false,
-      true
+      true,
+      false
     ]
   },
-  "hash": "4f6b3b472b4b78c0325cf3755f9ef1806d2e82328ceccbeade8cc2333c6dfe47"
+  "hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
 }

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration_ms",
-        "type_info": "Int4"
+        "type_info": "Int8"
       }
     ],
     "parameters": {

backend/.sqlx/query-3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, count(*) as count FROM queue WHERE\n        scheduled_for <= now() - ('3 seconds')::interval AND running = false\n        GROUP BY tag",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a"
+}

backend/.sqlx/query-47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f.json

@@ -1,15 +1,16 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
         "Int4",
+        "TextArray",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32"
+  "hash": "47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f"
 }

backend/.sqlx/query-502781c4e2fc692db7a66b850450a1934b2403a5cff0421fe9609f9b99d8ee95.json

@@ -11,7 +11,7 @@
     ],
     "parameters": {
       "Left": [
-        "Int4"
+        "Int8"
       ]
     },
     "nullable": [

backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json

@@ -67,6 +67,11 @@
         "ordinal": 12,
         "name": "openai_resource_path",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -87,7 +92,8 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
   "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"

backend/.sqlx/query-584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a"
+}

backend/.sqlx/query-5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password LIMIT $1 OFFSET $2",
+  "query": "SELECT email, login_type::text, verified, super_admin, name, company from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -49,5 +49,5 @@
       true
     ]
   },
-  "hash": "77ba7207c8f5fd7156542cfd9943aa9a9fa87a652131c261f5020bab9ba6b5a3"
+  "hash": "5f2cbdfe6eaf6cfaee1ec830678a5fdcf1024c8fab2b4a40d93dd5ffa4e7ba5c"
 }

backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Varchar",
-        "TextArray"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf"
-}

backend/.sqlx/query-6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03.json

@@ -1,14 +1,15 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET openai_resource_path = NULL WHERE workspace_id = $1",
+  "query": "UPDATE workspace_settings SET openai_resource_path = NULL, code_completion_enabled = $1 WHERE workspace_id = $2",
   "describe": {
     "columns": [],
     "parameters": {
       "Left": [
+        "Bool",
         "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "28eef409e9f25e4c037dd716d6515e96e8962acb59d5e64f9cc3227e0f5fd058"
+  "hash": "6268eabd561502a44e273d6391102278974623f7a7bcad6891d7abadf4d3ea03"
 }

backend/.sqlx/query-644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Json"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "644335d376b6554ab222d25dfa8722234661f2456b469f21b175e8607584614e"
+}

backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json

@@ -0,0 +1,74 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH uuid_table as (\n            select gen_random_uuid() as uuid from generate_series(1, $11)\n        )\n        INSERT INTO queue \n            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n            (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n        RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d"
+}

backend/.sqlx/query-6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6e6d781fe63602541f7cb4806d5ddde0792e1861ea8761b4875677c3d0594a86"
+}

backend/.sqlx/query-8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM password WHERE login_type != 'password'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483"
+}

backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = 'server'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+}

backend/.sqlx/query-8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM resource_type WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6"
+}

backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "avg_duration_s",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1"
-}

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -0,0 +1,66 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e"
+}

backend/.sqlx/query-9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47"
+}

backend/.sqlx/query-a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM\n                (SELECT duration_ms FROM completed_job WHERE script_path = $1\n                ORDER BY started_at\n                DESC LIMIT 10) AS t",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "avg_duration_s",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0"
+}

backend/.sqlx/query-a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "content",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "a2bc43114a2fb17fb62af19c34d8d5787ea304d1e8ef51f2c3d7b9882fdd0108"
+}

backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT progress::bigint FROM tutorial_progress WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "progress",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2"
+}

backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49"
+}

backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429"
+}

backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name LIKE 'worker__%'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7"
+}

backend/.sqlx/query-ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT openai_resource_path, code_completion_enabled FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "openai_resource_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      false
+    ]
+  },
+  "hash": "ceb97024ebf1a1c00ea1ed4952f66b229ca4622c537cc252d8ed52b4d24270ee"
+}

backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9"
+}

backend/.sqlx/query-d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4"
+}

backend/.sqlx/query-e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e40f7e0b61567f948bfea0b6f50518564634885ccc2c0d30ccca79fc13bdcf07"
+}

backend/.sqlx/query-e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468"
+}

backend/.sqlx/query-ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
+}

backend/.sqlx/query-efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "efbd9dc28ab5e53d070684b2e99d945872791171a1c8cef6088ea662f3b2cebb"
+}

backend/.sqlx/query-f516ca558816c2cbab3c8ae865ef8f764aa4686e7df9c53d30d49a3dbbf36af4.json

@@ -6,7 +6,7 @@
       {
         "ordinal": 0,
         "name": "duration",
-        "type_info": "Int8"
+        "type_info": "Numeric"
       }
     ],
     "parameters": {

backend/.sqlx/query-f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "database_length!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
+}

backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.168.3"
+version = "1.181.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.168.3"
+version = "1.181.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -54,7 +54,6 @@ tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
 git-version.workspace = true
-rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
@@ -64,6 +63,8 @@ once_cell.workspace = true
 prometheus.workspace = true
 uuid.workspace = true
 gethostname.workspace = true
+serde_json.workspace = true
+serde.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -96,7 +97,7 @@ tower = "^0"
 tower-http = { version = "^0", features = ["trace", "cors"] }
 tower-cookies = "^0"
 serde = "^1"
-serde_json = { version = "^1", features = ["preserve_order"] }
+serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
@@ -112,7 +113,7 @@ mime_guess = "^2"
 hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
-retainer = "^0"
+quick_cache = "^0"
 rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
@@ -155,7 +156,9 @@ sqlx = { version = "^0", features = [
     "chrono",
     "postgres",
     "runtime-tokio-rustls",
+    "bigdecimal"
 ] }
+bigdecimal = "^0"
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
@@ -187,6 +190,7 @@ postgres-native-tls = "^0"
 native-tls = "^0"
 samael = { version = "0.0.12", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
-rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
+rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
 pem = "3.0.1"
+nix = { version = "0.27.1", features = ["process", "signal"] }

backend/migrations/20230908155455_add_worker_group.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE worker_group_config;
+
+ALTER TABLE worker_ping DROP COLUMN worker_group;

backend/migrations/20230908155455_add_worker_group.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE worker_group_config (
+    name VARCHAR(255) PRIMARY KEY,
+    config JSONB DEFAULT '{}'::jsonb
+);
+
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS worker_group VARCHAR(255) NOT NULL DEFAULT 'default';
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS dedicated_worker VARCHAR(255);

backend/migrations/20230914062021_fix_group_grant_roles.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914062021_fix_group_grant_roles.up.sql

@@ -0,0 +1 @@
+-- Add up migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230914082838_relax_completed_job_constraints.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE completed_job ALTER COLUMN duration_ms TYPE bigint;
+ALTER TABLE completed_job ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN email TYPE VARCHAR(255);
+ALTER TABLE queue ALTER COLUMN canceled_by TYPE VARCHAR(255);

backend/migrations/20230915084031_add_themes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230915084031_add_themes.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;
+INSERT INTO resource (workspace_id, path, value, description, resource_type) SELECT id, 'f/app_themes/theme_0', '{"name": "Default Theme", "value": ""}', 'The default app theme', 'app_theme' FROM workspace ON CONFLICT DO NOTHING

backend/migrations/20230917075933_rename_config.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE IF exists config RENAME TO worker_group_config ;

backend/migrations/20230917075933_rename_config.up.sql

@@ -0,0 +1,29 @@
+-- Add up migration script here
+ALTER TABLE IF exists worker_group_config RENAME TO config;
+UPDATE config SET name = 'worker__' || name;
+
+CREATE FUNCTION "notify_config_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_config_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_config_change"
+ AFTER INSERT OR UPDATE ON "config"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_config_change" ();
+
+CREATE FUNCTION "notify_global_setting_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_global_setting_change"
+ AFTER INSERT OR UPDATE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_change" ();

backend/migrations/20230921084441_add_default_worker_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230921084441_add_default_worker_groups.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+INSERT INTO config (name, config) VALUES  
+    ('worker__default', '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun"]}'), 
+    ('worker__native', '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql", "snowflake"]}') ON CONFLICT DO NOTHING;

backend/migrations/20230927215813_add_app_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230927215813_add_app_groups.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;

backend/migrations/20230929022202_tutorials.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE tutorial_progress

backend/migrations/20230929022202_tutorials.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+CREATE TABLE  tutorial_progress(
+    email VARCHAR(255) PRIMARY KEY,
+    progress bit(64) NOT NULL DEFAULT B'0'
+);
+
+GRANT ALL ON tutorial_progress TO windmill_admin;
+GRANT ALL ON tutorial_progress TO windmill_user;

backend/migrations/20230929092213_copilot_code_completion.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN code_completion_enabled;

backend/migrations/20230929092213_copilot_code_completion.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN code_completion_enabled BOOLEAN NOT NULL DEFAULT false;

backend/migrations/20231002123723_notify_delete_config.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20231002123723_notify_delete_config.up.sql

@@ -0,0 +1,15 @@
+-- Add up migration script here
+
+CREATE FUNCTION "notify_global_setting_delete" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', OLD.name::text);
+    RETURN OLD;
+END;
+$$ LANGUAGE PLPGSQL;
+  
+CREATE OR REPLACE TRIGGER "notify_global_setting_delete"
+ AFTER DELETE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_delete" ();
+

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -35,6 +35,10 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "dateutil" => "python-dateutil",
     "mailparser" => "mail-parser",
     "mailparser-reply" => "mail-parser-reply",
+    "gitlab" => "python-gitlab",
+    "smbclient" => "smbprotocol",
+    "playhouse" => "peewee",
+    "dns" => "dnspython"
 };
 
 fn replace_import(x: String) -> String {

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -48,7 +48,7 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
 }
 
 lazy_static::lazy_static! {
-    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+))?"#).unwrap();
+    static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+(?:\[\])?))?"#).unwrap();
 
     // -- $1 name (type) = default
     static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *[\r\n$]"#).unwrap();
@@ -213,27 +213,35 @@ pub fn parse_mysql_typ(typ: &str) -> Typ {
 }
 
 pub fn parse_pg_typ(typ: &str) -> Typ {
-    match typ {
-        "varchar" => Typ::Str(None),
-        "text" => Typ::Str(None),
-        "int" => Typ::Int,
-        "bigint" => Typ::Int,
-        "bool" => Typ::Bool,
-        "char" => Typ::Str(None),
-        "smallint" => Typ::Int,
-        "smallserial" => Typ::Int,
-        "serial" => Typ::Int,
-        "bigserial" => Typ::Int,
-        "real" => Typ::Float,
-        "double precision" => Typ::Float,
-        "oid" => Typ::Int,
-        _ => Typ::Str(None),
+    if typ.ends_with("[]") {
+        let base_typ = parse_pg_typ(typ.strip_suffix("[]").unwrap());
+        Typ::List(Box::new(base_typ))
+    } else {
+        match typ {
+            "varchar" => Typ::Str(None),
+            "text" => Typ::Str(None),
+            "int" => Typ::Int,
+            "bigint" => Typ::Int,
+            "bool" => Typ::Bool,
+            "char" => Typ::Str(None),
+            "smallint" => Typ::Int,
+            "smallserial" => Typ::Int,
+            "serial" => Typ::Int,
+            "bigserial" => Typ::Int,
+            "real" => Typ::Float,
+            "double precision" => Typ::Float,
+            "numeric" => Typ::Float,
+            "decimal" => Typ::Float,
+            "oid" => Typ::Int,
+            "date" | "time" | "timestamp" => Typ::Datetime,
+            _ => Typ::Str(None),
+        }
     }
 }
 
 pub fn parse_bigquery_typ(typ: &str) -> Typ {
     if typ.ends_with("[]") {
-        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap_or(typ));
+        let base_typ = parse_bigquery_typ(typ.strip_suffix("[]").unwrap());
         Typ::List(Box::new(base_typ))
     } else {
         match typ {

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.154.3",
+  "version": "1.180.0",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -97,6 +97,15 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
 function addHeapObject(obj) {
     if (heap_next === heap.length) heap.push(heap.length + 1);
     const idx = heap_next;
@@ -115,15 +124,6 @@ function getFloat64Memory0() {
     return cachedFloat64Memory0;
 }
 
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
-}
-
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -492,13 +492,13 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
-    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
+    imports.wbg.__wbg_eval_2e5c54091872c19c = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -507,6 +507,10 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -543,10 +547,6 @@ function __wbg_get_imports() {
         const ret = BigInt.asUintN(64, arg0);
         return addHeapObject(ret);
     };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
         const ret = getObject(arg0) == getObject(arg1);
         return ret;

backend/src/ee.rs

@@ -1,41 +1,49 @@
 #[cfg(feature = "enterprise")]
-use base64::Engine;
-#[cfg(feature = "enterprise")]
-use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
-#[cfg(feature = "enterprise")]
-use sha2::Sha256;
+use windmill_common::error;
 
-#[cfg(feature = "enterprise")]
-pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
-    if let Some(license_key) = license_key {
-        let mut splitted_lk = license_key.split(".");
-        if splitted_lk.clone().count() != 3 {
-            panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
-        }
-        let id = splitted_lk.next().unwrap();
-        let expiry = splitted_lk.next().unwrap();
-        let signature_b64 = splitted_lk.next().unwrap();
+pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
+    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
 
-        let expiry_nb = expiry.parse::<u64>()?;
-        if expiry_nb < chrono::Utc::now().timestamp() as u64 {
-            panic!(
-                "License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
-                chrono::Utc::now().timestamp()
-            );
-        }
-        const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
-        let pub_key = rsa::RsaPublicKey::from_public_key_der(
-            &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
-        )?;
-        let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
-        rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
-            .verify(
-                &format!("{id}{expiry}").as_bytes(),
-                &rsa::pss::Signature::from(signature),
-            )
-            .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
-    } else {
-        panic!("License key is required for the enterprise edition");
+    let id = validate_license_key(license_key.clone()).await?;
+    {
+        let mut l = LICENSE_KEY_ID.write().await;
+        *l = id.to_string()
     }
+
+    {
+        let mut l = LICENSE_KEY.write().await;
+        *l = license_key
+    }
+    {
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = true
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn verify_license_key() -> error::Result<()> {
+    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::error::to_anyhow;
+
+    let expiry_nb = LICENSE_KEY
+        .read()
+        .await
+        .clone()
+        .split(".")
+        .nth(1)
+        .unwrap_or_else(|| "")
+        .parse::<u64>()
+        .map_err(to_anyhow)?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = false;
+    };
     Ok(())
 }

backend/src/main.rs

@@ -8,19 +8,27 @@
 
 use gethostname::gethostname;
 use git_version::git_version;
-use monitor::handle_zombie_jobs_periodically;
-use sqlx::{Pool, Postgres};
+use rand::Rng;
+use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     sync::Arc,
+    time::Duration,
 };
 use tokio::{
     fs::{metadata, DirBuilder},
-    join,
     sync::RwLock,
 };
-use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
+use windmill_common::{
+    global_settings::{
+        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, EXTRA_PIP_INDEX_URL_SETTING,
+        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
+    },
+    utils::rd_string,
+    worker::{reload_custom_tags_setting, WORKER_GROUP},
+    DB, METRICS_ADDR,
+};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -28,6 +36,12 @@ use windmill_worker::{
     PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
 
+use crate::monitor::{
+    initial_load, monitor_db, reload_base_url_setting, reload_extra_pip_index_url_setting,
+    reload_license_key, reload_npm_config_registry_setting, reload_retention_period_setting,
+    reload_server_config, reload_worker_config,
+};
+
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_PORT: u16 = 8000;
@@ -126,16 +140,9 @@ Windmill Community Edition {GIT_VERSION}
 
     display_config(&ENV_SETTINGS);
 
-    tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
-    if let Some(ref smtp) = *SMTP_CLIENT {
-        tracing::info!("Smtp client defined. Testing connection...");
-        if let Err(e) = smtp.connect().await {
-            tracing::error!("Failed to connect to smtp server: {}", e);
-        } else {
-            tracing::info!("Smtp client connected.");
-        }
-    }
-    if server_mode || num_workers > 0 {
+    let worker_mode = num_workers > 0;
+
+    if server_mode || worker_mode {
         let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
 
         let port = if server_mode {
@@ -144,6 +151,13 @@ Windmill Community Edition {GIT_VERSION}
             port_var.unwrap_or(0)
         };
 
+        // since it's only on server mode, the port is statically defined
+        let base_internal_url: String = format!("http://localhost:{}", port.to_string());
+
+        initial_load(&db, tx.clone(), worker_mode, server_mode).await;
+
+        monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
+
         if std::env::var("BASE_INTERNAL_URL").is_ok() {
             tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
         }
@@ -153,6 +167,12 @@ Windmill Community Edition {GIT_VERSION}
         let rsmq2 = rsmq.clone();
         let (port_tx, port_rx) = tokio::sync::oneshot::channel::<u16>();
 
+        DirBuilder::new()
+            .recursive(true)
+            .create("/tmp/windmill")
+            .await
+            .expect("could not create initial server dir");
+
         let server_f = async {
             windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe(), port_tx).await?;
             Ok(()) as anyhow::Result<()>
@@ -161,10 +181,11 @@ Windmill Community Edition {GIT_VERSION}
         let workers_f = async {
             let port = port_rx.await?;
             let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-            if num_workers > 0 {
+            if worker_mode {
                 run_workers(
                     db.clone(),
                     rx.resubscribe(),
+                    tx.clone(),
                     num_workers,
                     base_internal_url.clone(),
                     rsmq.clone(),
@@ -176,29 +197,130 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
-        let rsmq2 = rsmq.clone();
         let monitor_f = async {
-            if server_mode {
-                // since it's only on server mode, the port is statically defined
-                let base_internal_url: String = format!("http://localhost:{}", port.to_string());
-                monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2).await;
+            let db = db.clone();
+            let tx = tx.clone();
+            let rsmq = rsmq.clone();
+
+            let mut rx = rx.resubscribe();
+            let base_internal_url = base_internal_url.to_string();
+            let h = tokio::spawn(async move {
+                let mut listener = retry_listen_pg(&db).await;
+
+                loop {
+                    tokio::select! {
+                        _ = tokio::time::sleep(Duration::from_secs(30))    => {
+                            monitor_db(
+                                &db,
+                                &base_internal_url,
+                                rsmq.clone(),
+                                server_mode,
+                            )
+                            .await;
+                        },
+                        notification = listener.recv() => {
+                            match notification {
+                                Ok(n) => {
+                                    tracing::info!("Received new pg notification: {n:?}");
+                                    match n.channel() {
+                                        "notify_config_change" => {
+                                            match n.payload() {
+                                                "server" if server_mode => {
+                                                    tracing::info!("Server config change detected: {}", n.payload());
+
+                                                    reload_server_config(&db).await;
+                                                },
+                                                a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
+                                                    tracing::info!("Worker config change detected: {}", n.payload());
+                                                    reload_worker_config(&db, tx.clone(), true).await;
+                                                },
+                                                _ => {
+                                                    tracing::debug!("config changed but did not target this server/worker");
+                                                }
+                                            }
+                                        },
+                                        "notify_global_setting_change" => {
+                                            tracing::info!("Global setting change detected: {}", n.payload());
+                                            match n.payload() {
+                                                BASE_URL_SETTING => {
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload base url setting");
+                                                    }
+                                                },
+                                                OAUTH_SETTING => {
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload oauth setting");
+                                                    }
+                                                },
+                                                CUSTOM_TAGS_SETTING => {
+                                                    if let Err(e) = reload_custom_tags_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload custom tags setting");
+                                                    }
+                                                },
+                                                LICENSE_KEY_SETTING => {
+                                                    if let Err(e) = reload_license_key(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload license key setting");
+                                                    }
+                                                },
+                                                RETENTION_PERIOD_SECS_SETTING => {
+                                                    reload_retention_period_setting(&db).await
+                                                },
+                                                EXTRA_PIP_INDEX_URL_SETTING => {
+                                                    reload_extra_pip_index_url_setting(&db).await
+                                                },
+                                                NPM_CONFIG_REGISTRY_SETTING => {
+                                                    reload_npm_config_registry_setting(&db).await
+                                                },
+                                                REQUEST_SIZE_LIMIT_SETTING => {
+                                                    tracing::info!("Request limit size change detected, killing server expecting to be restarted");
+                                                    // we wait a bit randomly to avoid having all servers shutdown at same time
+                                                    let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                    if let Err(e) = tx.send(()) {
+                                                        tracing::error!(error = %e, "Could not send killpill to server");
+                                                    }
+                                                }
+                                                a @_ => {
+                                                    tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
+                                                }
+                                            }
+                                        },
+                                        _ => {
+                                            tracing::warn!("Unknown notification received");
+                                            continue;
+                                        }
+                                    }
+                                },
+                                Err(e) => {
+                                    tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
+                                    listener = retry_listen_pg(&db).await;
+                                    continue;
+                                }
+                            };
+                        },
+                        _ = rx.recv() => {
+                                println!("received killpill for monitor job");
+                                break;
+                        }
+                    }
+                }
+            });
+
+            if let Err(e) = h.await {
+                tracing::error!("Error waiting for monitor handle:{e}")
             }
             Ok(()) as anyhow::Result<()>
         };
 
         let metrics_f = async {
-            match metrics_addr {
-                Some(_addr) => {
-                    #[cfg(not(feature = "enterprise"))]
-                    panic!("Metrics are only available in the Enterprise Edition");
+            if let Some(_addr) = metrics_addr {
+                #[cfg(not(feature = "enterprise"))]
+                panic!("Metrics are only available in the Enterprise Edition");
 
-                    #[cfg(feature = "enterprise")]
-                    windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
-                        .await
-                        .map_err(anyhow::Error::from)
-                }
-                None => Ok(()),
+                #[cfg(feature = "enterprise")]
+                windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
             }
+            Ok(()) as anyhow::Result<()>
         };
 
         futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
@@ -208,6 +330,40 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
+async fn listen_pg(db: &DB) -> Option<PgListener> {
+    let mut listener = match PgListener::connect_with(&db).await {
+        Ok(l) => l,
+        Err(e) => {
+            tracing::error!(error = %e, "Could not connect to database");
+            return None;
+        }
+    };
+
+    if let Err(e) = listener
+        .listen_all(vec!["notify_config_change", "notify_global_setting_change"])
+        .await
+    {
+        tracing::error!(error = %e, "Could not listen to database");
+        return None;
+    }
+
+    return Some(listener);
+}
+
+async fn retry_listen_pg(db: &DB) -> PgListener {
+    let mut listener = listen_pg(db).await;
+    loop {
+        if listener.is_none() {
+            tracing::info!("Retrying listening to pg listen in 5 seconds");
+            tokio::time::sleep(Duration::from_secs(5)).await;
+            listener = listen_pg(db).await;
+        } else {
+            tracing::info!("Successfully connected to pg listen");
+            return listener.unwrap();
+        }
+    }
+}
+
 fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
@@ -225,40 +381,14 @@ fn display_config(envs: &[&str]) {
     )
 }
 
-pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
-    db: &Pool<Postgres>,
-    rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
-) -> tokio::task::JoinHandle<()> {
-    let db1 = db.clone();
-    let db2 = db.clone();
-
-    let rx2 = rx.resubscribe();
-    let base_internal_url = base_internal_url.to_string();
-    tokio::spawn(async move {
-        join!(
-            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
-            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
-        );
-    })
-}
-
 pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: Pool<Postgres>,
     rx: tokio::sync::broadcast::Receiver<()>,
+    tx: tokio::sync::broadcast::Sender<()>,
     num_workers: i32,
     base_internal_url: String,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
-    #[cfg(feature = "enterprise")]
-    ee::verify_license_key(LICENSE_KEY.clone())?;
-
-    #[cfg(not(feature = "enterprise"))]
-    if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) {
-        panic!("License key is required ONLY for the enterprise edition");
-    }
-
     let instance_name = gethostname()
         .to_str()
         .map(|x| {
@@ -320,6 +450,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
         let ip = ip.clone();
         let rx = rx.resubscribe();
+        let tx = tx.clone();
         let base_internal_url = base_internal_url.clone();
         let rsmq2 = rsmq.clone();
         let sync_barrier = sync_barrier.clone();
@@ -333,6 +464,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 num_workers as u32,
                 &ip,
                 rx,
+                tx,
                 &base_internal_url,
                 rsmq2,
                 sync_barrier,

backend/src/monitor.rs

@@ -1,18 +1,43 @@
-use std::time::Duration;
+use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration};
 
 use once_cell::sync::OnceCell;
+use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
-use tokio::sync::mpsc;
+use tokio::{
+    join,
+    sync::{mpsc, RwLock},
+};
 use uuid::Uuid;
+use windmill_api::{
+    oauth2::{build_oauth_clients, OAuthClient},
+    DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT,
+};
 use windmill_common::{
     error,
+    global_settings::{
+        BASE_URL_SETTING, EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        RETENTION_PERIOD_SECS_SETTING,
+    },
     jobs::{JobKind, QueuedJob},
-    METRICS_ENABLED,
+    server::load_server_config,
+    users::truncate_token,
+    worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
+    BASE_URL, DB, METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, NPM_CONFIG_REGISTRY,
+    PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
+#[cfg(feature = "enterprise")]
+use crate::ee::verify_license_key;
+
+#[cfg(feature = "enterprise")]
+use windmill_api::LICENSE_KEY_VALID;
+
+use crate::ee::set_license_key;
+
 lazy_static::lazy_static! {
     static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
     .ok()
@@ -35,27 +60,512 @@ lazy_static::lazy_static! {
         "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
     )
     .unwrap();
+
+    static ref QUEUE_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
+        "queue_count",
+        "Number of jobs in the queue",
+        &["tag"]
+    ).unwrap();
+
+    static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
+
 }
 
-pub async fn handle_zombie_jobs_periodically<
-    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
->(
+pub async fn initial_load(
     db: &Pool<Postgres>,
-    mut rx: tokio::sync::broadcast::Receiver<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
+    tx: tokio::sync::broadcast::Sender<()>,
+    worker_mode: bool,
+    server_mode: bool,
 ) {
-    loop {
-        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
-
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
-            _ = rx.recv() => {
-                    println!("received killpill for monitor job");
-                    break;
+    let reload_worker_config_f = async {
+        if worker_mode {
+            reload_worker_config(&db, tx, false).await;
+        }
+    };
+    let reload_custom_tags_f = async {
+        if server_mode {
+            if let Err(e) = reload_custom_tags_setting(db).await {
+                tracing::error!("Error reloading custom tags: {:?}", e)
             }
         }
+    };
+
+    let reload_base_url_f = async {
+        if let Err(e) = reload_base_url_setting(db).await {
+            tracing::error!("Error reloading base url: {:?}", e)
+        }
+    };
+
+    let reload_server_config_f = async {
+        if server_mode {
+            reload_server_config(&db).await;
+        }
+    };
+    let reload_retention_period_f = async {
+        if server_mode {
+            reload_retention_period_setting(&db).await;
+        }
+    };
+
+    let reload_request_size_f = async {
+        if server_mode {
+            reload_request_size(&db).await;
+        }
+    };
+
+    let reload_license_key_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = reload_license_key(&db).await {
+            tracing::error!("Error reloading license key: {:?}", e)
+        }
+    };
+
+    let reload_extra_pip_index_url_f = async {
+        if worker_mode {
+            reload_extra_pip_index_url_setting(&db).await;
+        }
+    };
+
+    let reload_npm_config_registry_f = async {
+        if worker_mode {
+            reload_npm_config_registry_setting(&db).await;
+        }
+    };
+
+    join!(
+        reload_worker_config_f,
+        reload_server_config_f,
+        reload_custom_tags_f,
+        reload_request_size_f,
+        reload_base_url_f,
+        reload_retention_period_f,
+        reload_license_key_f,
+        reload_extra_pip_index_url_f,
+        reload_npm_config_registry_f
+    );
+}
+
+pub async fn delete_expired_items(db: &DB) -> () {
+    let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
+        "DELETE FROM token WHERE expiration <= now()
+        RETURNING concat(substring(token for 10), '*****')",
+    )
+    .fetch_all(db)
+    .await;
+
+    match tokens_deleted_r {
+        Ok(tokens) => {
+            if tokens.len() > 0 {
+                tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
     }
+
+    let pip_resolution_r = sqlx::query_scalar!(
+        "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
+    )
+    .fetch_all(db)
+    .await;
+
+    match pip_resolution_r {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
+    }
+
+    let deleted_cache = sqlx::query_scalar!(
+            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
+        )
+        .fetch_all(db)
+        .await;
+
+    match deleted_cache {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} cache resource: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
+    }
+
+    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
+    if job_retention_secs > 0 {
+        let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                job_retention_secs
+            )
+            .fetch_all(db)
+            .await;
+
+        match deleted_jobs {
+            Ok(deleted_jobs) => {
+                if deleted_jobs.len() > 0 {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+                        deleted_jobs.len(),
+                        job_retention_secs,
+                        deleted_jobs,
+                    )
+                }
+            }
+            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+        }
+    }
+}
+
+pub async fn reload_extra_pip_index_url_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        EXTRA_PIP_INDEX_URL_SETTING,
+        "PIP_EXTRA_INDEX_URL",
+        PIP_EXTRA_INDEX_URL.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading extra_pip_index_url period: {:?}", e)
+    }
+}
+
+pub async fn reload_npm_config_registry_setting(db: &DB) {
+    if let Err(e) = reload_option_string_setting(
+        db,
+        NPM_CONFIG_REGISTRY_SETTING,
+        "NPM_CONFIG_REGISTRY",
+        NPM_CONFIG_REGISTRY.clone(),
+    )
+    .await
+    {
+        tracing::error!("Error reloading npm_config_registry period: {:?}", e)
+    }
+}
+
+pub async fn reload_retention_period_setting(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        RETENTION_PERIOD_SECS_SETTING,
+        "JOB_RETENTION_SECS",
+        60 * 60 * 24 * 60,
+        JOB_RETENTION_SECS.clone(),
+        |x| x,
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_request_size(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        REQUEST_SIZE_LIMIT_SETTING,
+        "REQUEST_SIZE_LIMIT",
+        DEFAULT_BODY_LIMIT,
+        REQUEST_SIZE_LIMIT.clone(),
+        |x| x.mul(1024 * 1024),
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_license_key(db: &DB) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        LICENSE_KEY_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var("LICENSE_KEY")
+        .ok()
+        .and_then(|x| x.parse::<String>().ok())
+        .unwrap_or(String::new());
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting LICENSE_KEY from db config: {}",
+                truncate_token(&v)
+            );
+            value = v;
+        } else {
+            tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value);
+        }
+    };
+
+    set_license_key(value).await?;
+
+    Ok(())
+}
+
+pub async fn reload_option_string_setting(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    lock: Arc<RwLock<Option<String>>>,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var).ok();
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = Some(v)
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        if value.is_none() {
+            tracing::info!("Loaded {setting_name} setting to None");
+        }
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    default: T,
+    lock: Arc<RwLock<T>>,
+    transformer: fn(T) -> T,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var)
+        .ok()
+        .and_then(|x| x.parse::<T>().ok())
+        .unwrap_or(default);
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<T>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = transformer(v);
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+    server_mode: bool,
+) {
+    let zombie_jobs_f = async {
+        if server_mode {
+            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+        }
+    };
+    let expired_items_f = async {
+        if server_mode {
+            delete_expired_items(&db).await;
+        }
+    };
+
+    let verify_license_key_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Err(e) = verify_license_key().await {
+            tracing::error!("Error verifying license key: {:?}", e);
+            let mut l = LICENSE_KEY_VALID.write().await;
+            *l = false;
+        } else {
+            let is_valid = LICENSE_KEY_VALID.read().await.clone();
+            if !is_valid {
+                let mut l = LICENSE_KEY_VALID.write().await;
+                *l = true;
+            }
+        }
+    };
+
+    let expose_queue_metrics_f = async {
+        if *METRICS_ENABLED && server_mode {
+            expose_queue_metrics(&db).await;
+        }
+    };
+    join!(
+        expired_items_f,
+        zombie_jobs_f,
+        expose_queue_metrics_f,
+        verify_license_key_f
+    );
+}
+
+pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
+    let queue_counts = sqlx::query!(
+        "SELECT tag, count(*) as count FROM queue WHERE
+        scheduled_for <= now() - ('3 seconds')::interval AND running = false
+        GROUP BY tag"
+    )
+    .fetch_all(db)
+    .await
+    .ok()
+    .unwrap_or_else(|| vec![]);
+    for q in queue_counts {
+        let count = q.count.unwrap_or(0);
+        let tag = q.tag;
+        let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+        metric.set(count as i64);
+    }
+}
+
+pub async fn reload_server_config(db: &Pool<Postgres>) {
+    let config = load_server_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading server config: {:?}", e)
+    } else {
+        let mut wc = SERVER_CONFIG.write().await;
+        tracing::info!("Reloading server config...");
+        *wc = config.unwrap()
+    }
+}
+
+pub async fn reload_worker_config(
+    db: &DB,
+    tx: tokio::sync::broadcast::Sender<()>,
+    kill_if_change: bool,
+) {
+    let config = load_worker_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading worker config: {:?}", e)
+    } else {
+        let wc = WORKER_CONFIG.read().await;
+        let config = config.unwrap();
+        if *wc != config {
+            if kill_if_change {
+                if (*wc).dedicated_worker != config.dedicated_worker {
+                    tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).init_bash != config.init_bash {
+                    tracing::info!("Init bash config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                }
+
+                if (*wc).cache_clear != config.cache_clear {
+                    tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send(());
+                    tracing::info!("Waiting 5 seconds to allow others workers to start potential jobs that depend on a potential shared cache volume");
+                    tokio::time::sleep(Duration::from_secs(5)).await;
+                    if let Err(e) = windmill_worker::common::clean_cache().await {
+                        tracing::error!("Error cleaning the cache: {e}");
+                    }
+                }
+            }
+            drop(wc);
+
+            let mut wc = WORKER_CONFIG.write().await;
+            tracing::info!("Reloading worker config...");
+            *wc = config
+        }
+    }
+}
+
+pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
+    let q_base_url = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        BASE_URL_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let base_url = if let Some(q) = q_base_url {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse base_url setting as a string, found: {:#?}",
+                &q.value
+            );
+            std::env::var("BASE_URL")
+                .ok()
+                .unwrap_or_else(|| "http://localhost".to_string())
+        }
+    } else {
+        std::env::var("BASE_URL")
+            .ok()
+            .unwrap_or_else(|| "http://localhost".to_string())
+    };
+
+    let q_oauth = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        OAUTH_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let oauths = if let Some(q) = q_oauth {
+        if let Ok(v) =
+            serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.value.clone())
+        {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse oauth setting as a json, found: {:#?}",
+                &q.value
+            );
+            None
+        }
+    } else {
+        None
+    };
+
+    let is_secure = base_url.starts_with("https://");
+
+    {
+        let mut l = OAUTH_CLIENTS.write().await;
+        *l = build_oauth_clients(&base_url, oauths)
+        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
+        .unwrap();
+    }
+
+    {
+        let mut l = BASE_URL.write().await;
+        *l = base_url
+    }
+
+    {
+        let mut l = IS_SECURE.write().await;
+        *l = is_secure;
+    }
+
+    Ok(())
 }
 
 async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(

backend/tests/fixtures/base.sql

@@ -17,59 +17,6 @@ INSERT INTO workspace_settings (workspace_id) VALUES
 
 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main(fail: boolean = true) {
-  if (fail) {
-    throw new Error("Failed")
-  }
-
-  return "OK"
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_script', -28028598712388162, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Error handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
-'',
-'',
-'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Recovery handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
-'',
-'',
-'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
-
-INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
-'test-workspace',
-'system',
-'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_flow'
-);
-
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 

backend/tests/fixtures/relative_bun.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'bun', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'bun', '');

backend/tests/fixtures/relative_deno.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'deno', '');

backend/tests/fixtures/relative_python.sql

@@ -0,0 +1,41 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system/same_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system_relative/different_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+from f.system.same_folder_script import main as test1
+from ..system.same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from .different_folder_script import main as test4
+
+def main():
+    return [test1(), test2(), test3(), test4()]
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'python3', '');

backend/tests/fixtures/schedule.sql

@@ -0,0 +1,52 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);

backend/tests/worker.rs

@@ -1,26 +1,81 @@
 use std::sync::Arc;
 
+#[cfg(feature = "enterprise")]
 use chrono::Timelike;
 use futures::StreamExt;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
-use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::{
-    sync::RwLock,
-    time::{timeout, Duration},
-};
-use windmill_api::jobs::{CompletedJob, Job};
+use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
+use tokio::sync::RwLock;
+
+#[cfg(feature = "enterprise")]
+use tokio::time::{timeout, Duration};
+
 use windmill_api_client::types::{
-    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+    CreateFlowBody, RawScript
 };
+
+use sqlx::query;
+
+#[cfg(feature = "enterprise")]
+use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
+
+use windmill_api_client::types::{NewScript, NewScriptLanguage};
+
+use windmill_common::worker::WORKER_CONFIG;
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
-    jobs::{JobPayload, RawCode},
-    scripts::ScriptLang,
+    jobs::{JobPayload, RawCode, JobKind},
+    scripts::{ScriptLang, ScriptHash}
 };
-use windmill_queue::{get_queued_job, PushIsolationLevel};
+use windmill_queue::PushIsolationLevel;
+use serde::Serialize;
+
+use std::str::FromStr;
+
+#[derive(Debug, sqlx::FromRow, Serialize)]
+pub struct CompletedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub started_at: chrono::DateTime<chrono::Utc>,
+    pub duration_ms: i64,
+    pub success: bool,
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    pub result: Option<serde_json::Value>,
+    pub logs: Option<String>,
+    pub deleted: bool,
+    pub raw_code: Option<String>,
+    pub canceled: bool,
+    pub canceled_by: Option<String>,
+    pub canceled_reason: Option<String>,
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    pub flow_status: Option<serde_json::Value>,
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    pub is_skipped: bool,
+    pub email: String,
+    pub visible_to_owner: bool,
+    pub mem_peak: Option<i32>,
+    pub tag: String,
+    pub script_hash: Option<ScriptHash>,
+    pub language: Option<ScriptLang>,
+    pub job_kind: JobKind,
+
+}
+
+impl CompletedJob {
+    pub fn json_result(&self) -> Option<serde_json::Value> {
+        self.result.clone()
+    }
+}
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -54,37 +109,6 @@ fn next_worker_name() -> String {
     format!("{id}/{thread_name}")
 }
 
-pub async fn get_job_by_id<'c>(
-    mut tx: Transaction<'c, Postgres>,
-    w_id: &str,
-    id: Uuid,
-) -> windmill_common::error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
-    let cjob_option = sqlx::query_as::<_, CompletedJob>(
-        "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-    )
-    .bind(id)
-    .bind(w_id)
-    .fetch_optional(&mut *tx)
-    .await?;
-    let job_option = match cjob_option {
-        Some(job) => Some(Job::CompletedJob(job)),
-        None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
-    };
-    if job_option.is_some() {
-        Ok((job_option, tx))
-    } else {
-        // check if a job had been moved in-between queries
-        let cjob_option = sqlx::query_as::<_, CompletedJob>(
-            "SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
-        )
-        .bind(id)
-        .bind(w_id)
-        .fetch_optional(&mut *tx)
-        .await?;
-        Ok((cjob_option.map(Job::CompletedJob), tx))
-    }
-}
-
 pub struct ApiServer {
     pub addr: std::net::SocketAddr,
     tx: tokio::sync::broadcast::Sender<()>,
@@ -115,21 +139,22 @@ impl ApiServer {
     }
 
     async fn close(self) -> anyhow::Result<()> {
+        println!("closing api server");
         let Self { tx, task, .. } = self;
         drop(tx);
         task.await.unwrap()
     }
 }
 
-async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
-    tracing::info!(
-        "{:#?}",
-        get_job_by_id(db.begin().await?, "test-workspace", id)
-            .await?
-            .0
-    );
-    Ok(())
-}
+// async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
+//     tracing::info!(
+//         "{:#?}",
+//         get_job_by_id(db.begin().await?, "test-workspace", id)
+//             .await?
+//             .0
+//     );
+//     Ok(())
+// }
 
 fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     cjob.flow_status.clone().and_then(|fs| {
@@ -299,7 +324,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!({
@@ -337,7 +362,7 @@ mod suspend_resume {
             .arg("port", json!(port))
             .run_until_complete(&db, port)
             .await
-            .result
+            .json_result()
             .unwrap();
 
         server.close().await.unwrap();
@@ -400,7 +425,7 @@ mod suspend_resume {
 
         server.close().await.unwrap();
 
-        let result = completed_job(flow, &db).await.result.unwrap();
+        let result = completed_job(flow, &db).await.json_result().unwrap();
 
         assert_eq!(
             json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
@@ -555,7 +580,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -584,7 +609,7 @@ def main(last, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await
-            .result
+            .json_result()
             .unwrap();
 
         assert_eq!(server.close().await, attempts);
@@ -626,7 +651,7 @@ def main(last, port):
             .run_until_complete(&db, server.addr.port())
             .await;
 
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(server.close().await, attempts);
         assert!(result["error"]
             .as_object()
@@ -690,7 +715,7 @@ def main(error, port):
             .arg("port", json!(server.addr.port()))
             .run_until_complete(&db, server.addr.port())
             .await;
-        let result = cjob.result.clone().unwrap();
+        let result = cjob.json_result().clone().unwrap();
         let failed_module = get_module(&cjob, "a").unwrap();
         match failed_module {
             FlowStatusModule::Failure { .. } => {}
@@ -748,7 +773,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -757,7 +782,7 @@ async fn test_iteration(db: Pool<Postgres>) {
         .arg("items", json!((0..257).collect::<Vec<_>>()))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
@@ -805,7 +830,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .arg("items", json!([]))
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([]));
 
@@ -815,7 +840,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
         .run_until_complete(&db, server.addr.port())
         .await;
     // println!("{:#?}", job);
-    let result = job.result.unwrap();
+    let result = job.json_result().unwrap();
     assert!(matches!(result, serde_json::Value::Array(_)));
     assert!(result[2]["error"]
         .as_object()
@@ -855,7 +880,7 @@ impl RunJob {
             args,
             /* user */ "test-user",
             /* email  */ "test@windmill.dev",
-            /* permissioned_as */ "u/admin".to_string(),
+            /* permissioned_as */ "u/test-user".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
@@ -881,7 +906,8 @@ impl RunJob {
         let uuid = self.push(db).await;
         let listener = listen_for_completed_jobs(db).await;
         in_test_worker(db, listener.find(&uuid), port).await;
-        completed_job(uuid, db).await
+        let r = completed_job(uuid, db).await;
+        r
     }
 }
 
@@ -916,13 +942,12 @@ async fn in_test_worker<Fut: std::future::Future>(
     };
 
     /* ensure the worker quits before we return */
-    drop(quit);
+    quit.send(()).expect("send");
 
     let _: () = worker
         .await
         .expect("worker timed out")
         .expect("worker panicked");
-
     res
 }
 
@@ -946,8 +971,13 @@ fn spawn_test_worker(
     let worker_name: String = next_worker_name();
     let ip: &str = Default::default();
 
+    let tx2 = tx.clone();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
+        {
+        let mut wc = WORKER_CONFIG.write().await;
+        (*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
+        }
         windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
             &db,
             worker_instance,
@@ -956,6 +986,7 @@ fn spawn_test_worker(
             1,
             ip,
             rx,
+            tx2,
             &base_internal_url,
             None,
             Arc::new(RwLock::new(None)),
@@ -995,8 +1026,8 @@ async fn listen_for_uuid_on(
 }
 
 async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
-    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-        .bind(uuid)
+
+    sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job  WHERE id = $1").bind(uuid)
         .fetch_one(db)
         .await
         .unwrap()
@@ -1106,7 +1137,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
         println!("deno flow iteration: {}", i);
         let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await;
         // println!("job: {:#?}", job.flow_status);
-        let result = job.result.unwrap();
+        let result = job.json_result().unwrap();
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i);
     }
 }
@@ -1142,7 +1173,7 @@ async fn test_identity(db: Pool<Postgres>) {
     let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
         .run_until_complete(&db, server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!(42));
 }
@@ -1331,7 +1362,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
 
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(
         result,
@@ -1386,7 +1417,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
     let job = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(result, serde_json::json!([[42]]));
 }
@@ -1431,7 +1462,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw 123"), result);
 
@@ -1440,7 +1471,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!(-123), result);
 }
 
@@ -1489,7 +1520,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .arg("n", json!(123))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!("last step saw [123]"), result);
 
@@ -1498,7 +1529,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
         .run_until_complete(&db, port)
         .await;
 
-    let result = cjob.result.unwrap();
+    let result = cjob.json_result().unwrap();
     assert_eq!(json!([-123]), result);
 }
 
@@ -1552,7 +1583,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}");
@@ -1587,7 +1618,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
             port,
         )
         .await
-        .result
+        .json_result()
         .unwrap();
 
         assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}");
@@ -1624,7 +1655,7 @@ func main(derp string) (string, error) {
     .arg("derp", json!("world"))
     .run_until_complete(&db, port)
     .await
-    .result
+    .json_result()
     .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1654,8 +1685,7 @@ echo "hello $msg"
     .arg("msg", json!("world"))
     .run_until_complete(&db, port)
     .await;
-
-    assert_eq!(job.result, Some(json!("hello world")));
+    assert_eq!(job.json_result(), Some(json!("hello world")));
 }
 
 #[sqlx::test(fixtures("base"))]
@@ -1682,7 +1712,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("hello world"));
@@ -1715,7 +1745,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(3));
@@ -1747,7 +1777,7 @@ def main():
 
     let result = run_job_in_new_worker_until_complete(&db, job, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!("test-workspace"));
@@ -1803,7 +1833,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(0));
@@ -1843,7 +1873,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
     let job = run_job_in_new_worker_until_complete(&db, flow, port).await;
 
     assert_eq!(
-        job.result.unwrap(),
+        job.json_result().unwrap(),
         serde_json::json!( {"error":  {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
     );
 }
@@ -1884,7 +1914,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([]));
@@ -1939,7 +1969,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!(9));
@@ -2007,7 +2037,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2]));
@@ -2043,7 +2073,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 3]));
@@ -2081,7 +2111,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2118,7 +2148,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
@@ -2165,7 +2195,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2202,7 +2232,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(
@@ -2266,7 +2296,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert_eq!(result, serde_json::json!([1, 2, 3]));
@@ -2323,7 +2353,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
     let flow = JobPayload::RawFlow { value: flow, path: None };
     let result = run_job_in_new_worker_until_complete(&db, flow, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     println!("{:#?}", result);
@@ -2388,7 +2418,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(0))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2404,7 +2434,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(1))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2420,7 +2450,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(2))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
 
     assert!(result["from failure module"]["error"]
@@ -2436,7 +2466,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
         .arg("n", json!(3))
         .run_until_complete(&db, port)
         .await
-        .result
+        .json_result()
         .unwrap();
     assert_eq!(json!({ "l": [0, 1, 2] }), result);
 }
@@ -2586,7 +2616,9 @@ async fn test_rust_client(db: Pool<Postgres>) {
     .unwrap();
 }
 
-#[sqlx::test(fixtures("base"))]
+
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_script_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2642,8 +2674,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2702,8 +2733,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2721,7 +2751,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
 }
 
 
-#[sqlx::test(fixtures("base"))]
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2778,8 +2809,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
 
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2839,8 +2869,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
             let uuid = uuid.unwrap().unwrap();
             
             let completed_job =
-                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
-                    .bind(uuid)
+                query!("SELECT script_path FROM completed_job  WHERE id = $1", uuid)
                     .fetch_one(&db2)
                     .await
                     .unwrap();
@@ -2856,3 +2885,203 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     )
     .await;
 }
+
+
+async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    client.create_script(
+        "test-workspace",
+        &NewScript {
+            language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
+            content: script_content,
+            path: "f/system/test_import".to_string(),
+            concurrent_limit: vec![],
+            concurrency_time_window_s: vec![],
+            cache_ttl: None,
+            dedicated_worker: None,
+            description: "".to_string(),
+            draft_only: None,
+            envs: vec![],
+            is_template: None,
+            kind: None,
+            parent_hash: None,
+            lock: vec![],
+            summary: "".to_string(),
+            tag: None,
+            schema: std::collections::HashMap::new(),
+        },
+    ).await.unwrap();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        completed.next().await; // deployed script
+
+        let script =
+            query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string())
+                .fetch_one(&db2)
+                .await
+                .unwrap();
+
+        let job = RunJob::from(JobPayload::ScriptHash {
+            path: "f/system/test_import".to_string(),
+            hash: ScriptHash(script.hash),
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+            dedicated_worker: None,
+            language,
+        }).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        let job = RunJob::from(JobPayload::Code(RawCode {
+            content: script_content,
+            path: Some("f/system/test_import".to_string()),
+            language,
+            lock: None,
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+        })).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_relative_imports_bun(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_nested_imports_bun(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_relative_imports_deno(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_nested_imports_deno(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_relative_imports_python(db: Pool<Postgres>) {
+    let content = r#"
+from f.system.same_folder_script import main as test1
+from .same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from ..system_relative.different_folder_script import main as test4
+    
+def main():
+    return [test1(), test2(), test3(), test4()]
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_nested_imports_python(db: Pool<Postgres>) {
+
+    let content = r#"
+
+from f.system_relative.nested_script import main as test
+
+def main():
+    return test()
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+}

backend/windmill-api/Cargo.toml

@@ -53,7 +53,7 @@ cron.workspace = true
 mime_guess.workspace = true
 rust-embed.workspace = true
 tracing-subscriber.workspace = true
-retainer.workspace = true
+quick_cache.workspace = true
 rand.workspace = true
 time.workspace = true
 magic-crypt.workspace = true
@@ -74,3 +74,5 @@ bytes.workspace = true
 mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
+rsa.workspace = true
+uuid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.168.3
+  version: 1.181.0
   title: Windmill API
 
   contact:
@@ -575,6 +575,81 @@ paths:
             application/json:
               schema: {}
 
+  /settings/test_smtp:
+    post:
+      summary: test smtp
+      operationId: testSmtp
+      tags:
+        - setting
+      requestBody:
+        description: test smtp payload
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                to:
+                  type: string
+                smtp:
+                  type: object
+                  properties:
+                    host:
+                      type: string
+                    username:
+                      type: string
+                    password:
+                      type: string
+                    port:
+                      type: integer
+                    from:
+                      type: string
+                    tls_implicit:
+                      type: boolean
+                  required:
+                    - host
+                    - username
+                    - password
+                    - port
+                    - from
+                    - tls_implicit
+              required:
+                - to
+                - smtp
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
+  /settings/test_license_key:
+    post:
+      summary: test license key
+      operationId: testLicenseKey
+      tags:
+        - setting
+      requestBody:
+        description: test license key
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                license_key:
+                  type: string
+              required:
+                - license_key
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -603,6 +678,45 @@ paths:
               schema:
                 type: string
 
+  /users/tutorial_progress:
+    get:
+      summary: get tutorial progress
+      operationId: getTutorialProgress
+      tags:
+        - user
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  progress:
+                    type: integer
+    post:
+      summary: update tutorial progress
+      operationId: updateTutorialProgress
+      tags:
+        - user
+      requestBody:
+        description: progress update
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                progress:
+                  type: integer
+      responses:
+        "200":
+          description: tutorial progress
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /users/usage:
     get:
       summary: get current usage outside of premium workspaces
@@ -1040,8 +1154,12 @@ paths:
                     type: string
                   openai_resource_path:
                     type: string
+                  code_completion_enabled:
+                    type: boolean
                   error_handler:
                     type: string
+                required:
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/get_deploy_to:
     get:
@@ -1194,24 +1312,28 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/edit_openai_resource_path:
+  /w/{workspace}/workspaces/edit_copilot_config:
     post:
-      summary: edit OpenAI resource path
-      operationId: editOpenaiResourcePath
+      summary: edit copilot config
+      operationId: editCopilotConfig
       tags:
         - workspace
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: WorkspaceOpenaiResourcePath
+        description: WorkspaceCopilotConfig
         required: true
         content:
           application/json:
             schema:
               type: object
+              required:
+                - code_completion_enabled
               properties:
                 openai_resource_path:
                   type: string
+                code_completion_enabled:
+                  type: boolean
       responses:
         "200":
           description: status
@@ -1220,10 +1342,10 @@ paths:
               schema:
                 type: string
 
-  /w/{workspace}/workspaces/exists_openai_resource_path:
+  /w/{workspace}/workspaces/get_copilot_info:
     get:
-      summary: OpenAI resource path exists
-      operationId: existsOpenaiResourcePath
+      summary: get copilot info
+      operationId: getCopilotInfo
       tags:
         - workspace
       parameters:
@@ -1235,7 +1357,15 @@ paths:
           content:
             text/plain:
               schema:
-                type: boolean
+                type: object
+                properties:
+                  exists_openai_resource_path:
+                    type: boolean
+                  code_completion_enabled:
+                    type: boolean
+                required:
+                  - exists_openai_resource_path
+                  - code_completion_enabled
 
   /w/{workspace}/workspaces/edit_error_handler:
     post:
@@ -1934,6 +2064,12 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/Path"
+        - name: job_id
+          description: job id
+          in: query
+          schema:
+            type: string
+            format: uuid
       responses:
         "200":
           description: resource value
@@ -2004,6 +2140,31 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableResource"
 
+  /w/{workspace}/resources/list_search:
+    get:
+      summary: list resources for search
+      operationId: listSearchResource
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: resource list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/resources/list_names/{name}:
     get:
       summary: list resource names
@@ -2386,6 +2547,72 @@ paths:
                   - content
                   - language
 
+  /scripts/hub/query:
+    get:
+      summary: query hub scripts by similarity
+      operationId: queryHubScripts
+      tags:
+        - script
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: kind
+          description: query scripts kind
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
+  /w/{workspace}/scripts/list_search:
+    get:
+      summary: list scripts for search
+      operationId: listSearchScript
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: script list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    content:
+                      type: string
+                  required:
+                    - path
+                    - content
+
   /w/{workspace}/scripts/list:
     get:
       summary: list all available scripts
@@ -3031,6 +3258,31 @@ paths:
                 items:
                   type: string
 
+  /w/{workspace}/flows/list_search:
+    get:
+      summary: list flows for search
+      operationId: listSearchFlow
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: flow list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/flows/list:
     get:
       summary: list all available flows
@@ -3334,6 +3586,31 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/apps/list_search:
+    get:
+      summary: list apps for search
+      operationId: listSearchApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: app list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    path:
+                      type: string
+                    value: {}
+                  required:
+                    - path
+                    - value
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -3479,6 +3756,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
+  /w/{workspace}/apps_u/public_resource/{path}:
+    get:
+      summary: get public resource
+      operationId: get public resource
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -3914,6 +4207,27 @@ paths:
                 required:
                   - database_length
 
+  /w/{workspace}/jobs/completed/count:
+    get:
+      summary: get completed count
+      operationId: getCompletedCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: completed count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
   /w/{workspace}/jobs/queue/cancel_all:
     post:
       summary: cancel all jobs
@@ -3991,6 +4305,7 @@ paths:
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/CreatedOrStartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
@@ -5191,6 +5506,80 @@ paths:
                 items:
                   $ref: "#/components/schemas/WorkerPing"
 
+  /configs/list_worker_groups:
+    get:
+      summary: list worker groups
+      operationId: listWorkerGroups
+      tags:
+        - config
+      responses:
+        "200":
+          description: a list of worker group configs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    config: {}
+                  required:
+                    - name
+                    - config
+
+  /configs/get/{name}:
+    get:
+      summary: get config
+      operationId: get config
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: a config
+          content:
+            application/json:
+              schema: {}
+
+  /configs/update/{name}:
+    post:
+      summary: Update config
+      operationId: updateConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      requestBody:
+        description: worker group
+        required: true
+        content:
+          application/json:
+            schema: {}
+      responses:
+        "200":
+          description: Update a worker group
+          content:
+            text/plain:
+              schema:
+                type: string
+    delete:
+      summary: Delete Config
+      operationId: deleteConfig
+      tags:
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: Delete config
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/acls/get/{kind}/{path}:
     get:
       summary: get granular acls
@@ -6616,7 +7005,7 @@ components:
             - "workspaces.edit_deploy_to"
             - "workspaces.edit_auto_invite_domain"
             - "workspaces.edit_webhook"
-            - "workspaces.edit_openai_resource_path"
+            - "workspaces.edit_copilot_config"
             - "workspaces.edit_error_handler"
             - "workspaces.create"
             - "workspaces.update"
@@ -7111,6 +7500,8 @@ components:
           type: array
           items:
             type: string
+        worker_group:
+          type: string
       required:
         - worker
         - worker_instance
@@ -7118,6 +7509,7 @@ components:
         - started_at
         - ip
         - jobs_executed
+        - worker_group
 
     UserWorkspaceList:
       type: object

backend/windmill-api/src/apps.rs

@@ -43,6 +43,7 @@ use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
+        .route("/list_search", get(list_search_apps))
         .route("/get/p/*path", get(get_app))
         .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
@@ -57,6 +58,7 @@ pub fn unauthed_service() -> Router {
     Router::new()
         .route("/execute_component/*path", post(execute_component))
         .route("/public_app/:secret", get(get_public_app_by_secret))
+        .route("/public_resource/*path", get(get_public_resource))
 }
 
 pub fn global_service() -> Router {
@@ -159,6 +161,38 @@ pub struct EditApp {
     pub policy: Option<Policy>,
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchApp {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_apps(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchApp>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchApp,
+        "SELECT path, app_version.value from app LEFT JOIN app_version ON app_version.id = versions[array_upper(versions, 1)]  WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_apps(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -341,6 +375,27 @@ async fn get_public_app_by_secret(
     Ok(Json(app))
 }
 
+async fn get_public_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    if !path.starts_with("f/app_themes/") {
+        return Err(Error::BadRequest(
+            "Only app themes are public resources".to_string(),
+        ));
+    }
+    let res = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten();
+    Ok(Json(res))
+}
+
 async fn get_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -508,6 +563,7 @@ pub async fn get_hub_app_by_id(
         &format!("https://hub.windmill.dev/apps/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()

backend/windmill-api/src/capture.rs

@@ -9,7 +9,7 @@
 use axum::{
     extract::{Extension, Path, Query},
     routing::{get, post, put},
-    Json, Router,
+    Router,
 };
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
@@ -21,7 +21,7 @@ use windmill_common::{
 
 use crate::{
     db::{ApiAuthed, DB},
-    jobs::add_include_headers,
+    jobs::{add_include_headers, add_raw_string, JsonOrForm},
 };
 
 const KEEP_LAST: i64 = 8;
@@ -95,11 +95,12 @@ pub async fn update_payload(
     Path((w_id, path)): Path<(String, StripPath)>,
     Query(run_query): Query<IncludeHeaderQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
-
     let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
+
     sqlx::query!(
         "
        UPDATE capture

backend/windmill-api/src/configs.rs

@@ -0,0 +1,109 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use axum::{
+    extract::{Extension, Path},
+    routing::{get, post},
+    Json, Router,
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use windmill_common::{
+    db::UserDB,
+    error::{self},
+    DB,
+};
+
+use crate::{db::ApiAuthed, utils::require_super_admin};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/list_worker_groups", get(list_worker_groups))
+        .route("/update/:name", post(update_config).delete(delete_config))
+        .route("/get/:name", get(get_config))
+}
+
+#[derive(Serialize, Deserialize, FromRow)]
+struct Config {
+    name: String,
+    config: serde_json::Value,
+}
+
+async fn list_worker_groups(Extension(db): Extension<DB>) -> error::JsonResult<Vec<Config>> {
+    let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+        .fetch_all(&db)
+        .await?;
+    Ok(Json(rows))
+}
+
+async fn get_config(
+    authed: ApiAuthed,
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Option<serde_json::Value>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
+        .fetch_optional(&db)
+        .await?
+        .map(|c| c.config);
+
+    Ok(Json(config))
+}
+
+async fn update_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(config): Json<serde_json::Value>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    #[cfg(not(feature = "enterprise"))]
+    if name.starts_with("worker__") {
+        return Err(error::Error::BadRequest(
+            "Worker groups configurable from UI available only in the enterprise version"
+                .to_string(),
+        ));
+    }
+
+    sqlx::query!(
+        "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+        &name,
+        config
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Updated config {name}"))
+}
+
+async fn delete_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    let tx = user_db.begin(&authed).await?;
+
+    require_super_admin(&db, &authed.email).await?;
+    tx.commit().await?;
+
+    let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name)
+        .fetch_all(&db)
+        .await?;
+
+    if deleted.len() == 0 {
+        return Err(error::Error::NotFound(format!(
+            "Config {name} not found",
+            name = name
+        )));
+    }
+    Ok(format!("Deleted config {name}"))
+}

backend/windmill-api/src/ee.rs

@@ -0,0 +1,41 @@
+use anyhow::anyhow;
+use base64::Engine;
+use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
+use sha2::Sha256;
+
+pub async fn validate_license_key(license_key: String) -> anyhow::Result<String> {
+    let mut splitted_lk = license_key.split(".");
+    if splitted_lk.clone().count() != 3 {
+        return Err(anyhow!(
+            "license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)"
+        ));
+    }
+
+    let id = splitted_lk.next().unwrap();
+    let expiry = splitted_lk.next().unwrap();
+    let signature_b64 = splitted_lk.next().unwrap();
+
+    const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
+    let pub_key = rsa::RsaPublicKey::from_public_key_der(
+        &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
+    )?;
+    let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
+    rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
+        .verify(
+            &format!("{id}{expiry}").as_bytes(),
+            &rsa::pss::Signature::from(signature),
+        )
+        .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
+
+    let expiry_nb = expiry.parse::<u64>()?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        return Err(anyhow!("License key expired".to_string()));
+    };
+
+    Ok(id.to_string())
+}

backend/windmill-api/src/flows.rs

@@ -24,7 +24,7 @@ use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
-use sqlx::{Postgres, Transaction};
+use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     db::UserDB,
@@ -42,6 +42,7 @@ use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, Que
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_flows))
+        .route("/list_search", get(list_search_flows))
         .route("/create", post(create_flow))
         .route("/update/*path", post(update_flow))
         .route("/archive/*path", post(archive_flow_by_path))
@@ -58,6 +59,38 @@ pub fn global_service() -> Router {
         .route("/hub/get/:id", get(get_hub_flow_by_id))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchFlow {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_flows(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchFlow>> {
+    let mut tx = user_db.begin(&authed).await?;
+
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchFlow,
+        "SELECT path, value from flow WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_flows(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -160,6 +193,7 @@ pub async fn get_hub_flow_by_id(
         &format!("https://hub.windmill.dev/flows/{id}/json"),
         &email,
         false,
+        None,
     )
     .await?
     .json()

backend/windmill-api/src/groups.rs

@@ -15,6 +15,7 @@ use axum::{
     Json, Router,
 };
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::worker::CLOUD_HOSTED;
 use windmill_common::{db::UserDB, users::username_to_permissioned_as};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -23,7 +24,6 @@ use windmill_common::{
 
 use serde::{Deserialize, Serialize};
 use sqlx::{query_scalar, FromRow, Postgres, Transaction};
-use windmill_queue::CLOUD_HOSTED;
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -212,6 +212,7 @@ async fn _check_nb_of_groups(db: &DB) -> Result<()> {
     }
     return Ok(());
 }
+
 async fn create_group(
     authed: ApiAuthed,
     Extension(_db): Extension<DB>,

backend/windmill-api/src/inputs.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{db::ApiAuthed, jobs::CompletedJob};
+use crate::db::ApiAuthed;
 use axum::{
     extract::{Path, Query},
     routing::{get, post},
@@ -15,7 +15,7 @@ use axum::{
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use sqlx::types::Uuid;
+use sqlx::{types::Uuid, FromRow};
 use std::{
     fmt::{Display, Formatter},
     vec,
@@ -102,6 +102,15 @@ pub struct Input {
     success: bool,
 }
 
+#[derive(Debug, Serialize, Deserialize, FromRow)]
+pub struct CompletedJobMini {
+    id: Uuid,
+    created_at: chrono::DateTime<chrono::Utc>,
+    args: Option<serde_json::Value>,
+    created_by: String,
+    success: bool,
+}
+
 async fn get_input_history(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -114,13 +123,13 @@ async fn get_input_history(
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = &format!(
-        "select * from completed_job \
+        "select id, created_at, created_by, args, success from completed_job \
         where {} = $1 and job_kind = $2 and workspace_id = $3 \
         order by created_at desc limit $4 offset $5",
         r.runnable_type.column_name()
     );
 
-    let query = sqlx::query_as::<_, CompletedJob>(sql);
+    let query = sqlx::query_as::<_, CompletedJobMini>(sql);
 
     let query = match r.runnable_type {
         RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),

backend/windmill-api/src/lib.rs

@@ -11,9 +11,8 @@ use crate::oauth2::AllClients;
 use crate::saml::{SamlSsoLogin, ServiceProviderExt};
 use crate::scim::has_scim_token;
 use crate::tracing_init::MyOnFailure;
-use crate::workers::ALL_TAGS;
 use crate::{
-    oauth2::{build_oauth_clients, SlackVerifier},
+    oauth2::SlackVerifier,
     tracing_init::{MyMakeSpan, MyOnResponse},
     users::OptAuthed,
     webhook_util::WebhookShared,
@@ -25,9 +24,10 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
 use hyper::{http, Method};
-use mail_send::SmtpClientBuilder;
 use reqwest::Client;
+use std::collections::HashMap;
 use std::{net::SocketAddr, sync::Arc};
+use tokio::sync::RwLock;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
 use tower_http::{
@@ -36,14 +36,18 @@ use tower_http::{
 };
 use windmill_common::db::UserDB;
 use windmill_common::utils::rd_string;
+use windmill_common::worker::ALL_TAGS;
+use windmill_common::BASE_URL;
 
 use windmill_common::error::AppError;
 
 mod apps;
 mod audit;
 mod capture;
+mod configs;
 mod db;
 mod drafts;
+pub mod ee;
 mod favorite;
 mod flows;
 mod folders;
@@ -51,7 +55,7 @@ mod granular_acls;
 mod groups;
 mod inputs;
 pub mod jobs;
-mod oauth2;
+pub mod oauth2;
 mod openai;
 mod raw_apps;
 mod resources;
@@ -72,16 +76,11 @@ mod workspaces;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-pub use users::delete_expired_items_perdiodically;
+pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
 
-pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
 lazy_static::lazy_static! {
-    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
-    pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
-    .ok()
-    .and_then(|x| x.parse::<usize>().ok())
-    .unwrap_or(DEFAULT_BODY_LIMIT);
+    pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
 
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
@@ -90,59 +89,22 @@ lazy_static::lazy_static! {
         .ok()
         .map(|x| SlackVerifier::new(x).unwrap());
 
-        static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
+    pub static ref IS_SECURE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
 
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
         .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
         .build().unwrap();
 
-    pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
-        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
-        .unwrap();
+    pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
+        logins: HashMap::new(),
+        connects: HashMap::new(),
+        slack: None
+    }));
 
-    pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
-        let smtp = parse_smtp();
-        if let Some(smtp) = smtp {
-            match smtp {
-                Ok(smtp) => Some(smtp),
-                Err(e) => {
-                    tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e);
-                    None
-                }
-            }
-        } else {
-            tracing::warn!("SMTP is not configured, emails will not be sent");
-            None
-        }
-    };
-
-    pub static ref SMTP_FROM: String = std::env::var("SMTP_FROM").unwrap_or_else(|_| "noreply@getwindmill.com".to_string());
-
-    pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
-}
-
-pub fn parse_smtp() -> Option<windmill_common::error::Result<SmtpClientBuilder<String>>> {
-    let username = std::env::var("SMTP_USERNAME").ok();
-    let port = std::env::var("SMTP_PORT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(587);
-    let password = std::env::var("SMTP_PASSWORD").ok();
-    let host = std::env::var("SMTP_HOST").ok();
-    let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(false);
-
-    if username.is_some() && password.is_some() && host.is_some() {
-        let smtp = SmtpClientBuilder::new(host.unwrap(), port)
-            .implicit_tls(tls_implicit)
-            .credentials((username.unwrap(), password.unwrap()));
-        Some(Ok(smtp))
-    } else {
-        None
-    }
+    pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+    pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+    pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 }
 
 pub async fn run_server(
@@ -153,7 +115,7 @@ pub async fn run_server(
     port_tx: tokio::sync::oneshot::Sender<u16>,
 ) -> anyhow::Result<()> {
     if let Some(mut rsmq) = rsmq.clone() {
-        for tag in ALL_TAGS.clone() {
+        for tag in ALL_TAGS.read().await.iter() {
             let r =
                 rsmq_async::RsmqConnection::create_queue(&mut rsmq, &tag, None, None, None).await;
             if let Err(e) = r {
@@ -185,7 +147,9 @@ pub async fn run_server(
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
-        .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
+        .layer(DefaultBodyLimit::max(
+            REQUEST_SIZE_LIMIT.read().await.clone(),
+        ));
 
     let cors = CorsLayer::new()
         .allow_methods([Method::GET, Method::POST])
@@ -239,6 +203,7 @@ pub async fn run_server(
                 .nest("/settings", settings::global_service())
                 .nest("/jobs", jobs::global_root_service())
                 .nest("/workers", workers::global_service())
+                .nest("/configs", configs::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/groups", groups::global_service())
                 .nest("/flows", flows::global_service())
@@ -287,15 +252,16 @@ pub async fn run_server(
 
     let instance_name = rd_string(5);
 
-    tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening");
     let server = axum::Server::bind(&addr).serve(app.into_make_service());
 
     let port = server.local_addr().port();
     tracing::info!(
+        instance = %instance_name,
         "server started on port={} and addr={}",
         port,
         server.local_addr().ip()
     );
+
     port_tx
         .send(server.local_addr().port())
         .expect("Failed to send port");
@@ -305,8 +271,6 @@ pub async fn run_server(
         println!("Graceful shutdown of server");
     });
 
-    tokio::spawn(async move { auth_cache.monitor().await });
-
     server.await?;
     Ok(())
 }
@@ -354,13 +318,7 @@ async fn ee_license() -> &'static str {
 
 #[cfg(feature = "enterprise")]
 async fn ee_license() -> String {
-    LICENSE_KEY
-        .as_ref()
-        .unwrap()
-        .split(".")
-        .next()
-        .unwrap()
-        .to_string()
+    LICENSE_KEY_ID.read().await.clone()
 }
 
 async fn openapi() -> &'static str {

backend/windmill-api/src/oauth2.rs

@@ -35,6 +35,7 @@ use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::db::UserDB;
 use windmill_common::jobs::JobPayload;
+use windmill_common::more_serde::maybe_number_opt;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
@@ -81,7 +82,7 @@ pub fn workspaced_service() -> Router {
         .route("/connect_slack_callback", post(connect_slack_callback))
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ClientWithScopes {
     client: OClient,
     scopes: Vec<String>,
@@ -120,7 +121,10 @@ pub struct AllClients {
     pub slack: Option<OClient>,
 }
 
-pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
+pub fn build_oauth_clients(
+    base_url: &str,
+    oauths_from_config: Option<HashMap<String, OAuthClient>>,
+) -> anyhow::Result<AllClients> {
     let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
         "../../oauth_connect.json"
     ))?;
@@ -128,26 +132,36 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         "../../oauth_login.json"
     ))?;
 
-    let path = "./oauth.json";
-    let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
-        str::from_utf8(
-            &base64::engine::general_purpose::STANDARD
-                .decode(e)
-                .map_err(to_anyhow)?,
-        )?
-        .to_string()
-    } else if std::path::Path::new(path).exists() {
-        fs::read_to_string(path).map_err(to_anyhow)?
+    let oauths = if let Some(oauths) = oauths_from_config {
+        oauths
     } else {
-        tracing::warn!("oauth.json not found, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
+        let path = "./oauth.json";
+        let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
+            str::from_utf8(
+                &base64::engine::general_purpose::STANDARD
+                    .decode(e)
+                    .map_err(to_anyhow)?,
+            )?
+            .to_string()
+        } else if std::path::Path::new(path).exists() {
+            fs::read_to_string(path).map_err(to_anyhow)?
+        } else {
+            tracing::warn!("oauth.json not found, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
 
-    if content.is_empty() {
-        tracing::warn!("oauth.json is empty, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
-    let oauths: HashMap<String, OAuthClient> =
+        if content.is_empty() {
+            tracing::warn!("oauth.json is empty, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
         match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
             Ok(clients) => clients,
             Err(e) => {
@@ -156,7 +170,8 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             }
         }
         .into_iter()
-        .collect();
+        .collect()
+    };
 
     tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", "));
 
@@ -168,7 +183,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -177,17 +192,25 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 base_url,
                 None,
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: client_params.allowed_domains.clone(),
-                    userinfo_url: config.userinfo_url,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: client_params.allowed_domains.clone(),
+                            userinfo_url: config.userinfo_url,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
@@ -199,7 +222,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -212,43 +235,61 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                     None
                 },
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: None,
-                    userinfo_url: None,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: None,
+                            userinfo_url: None,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
-    let slack = oauths.get("slack").map(|v| {
-        build_basic_client(
-            "slack".to_string(),
-            OAuthConfig {
-                auth_url: "https://slack.com/oauth/authorize".to_string(),
-                token_url: "https://slack.com/api/oauth.access".to_string(),
-                userinfo_url: None,
-                scopes: None,
-                extra_params: None,
-                extra_params_callback: None,
-                req_body_auth: None,
-            },
-            v.clone(),
-            false,
-            base_url,
-            Some(format!("{base_url}/oauth/callback_slack")),
-        )
-        .1
-    });
-
-    Ok(AllClients { logins, connects, slack })
+    let slack = oauths
+        .get("slack")
+        .map(|v| {
+            build_basic_client(
+                "slack".to_string(),
+                OAuthConfig {
+                    auth_url: "https://slack.com/oauth/authorize".to_string(),
+                    token_url: "https://slack.com/api/oauth.access".to_string(),
+                    userinfo_url: None,
+                    scopes: None,
+                    extra_params: None,
+                    extra_params_callback: None,
+                    req_body_auth: None,
+                },
+                v.clone(),
+                false,
+                base_url,
+                Some(format!("{base_url}/oauth/callback_slack")),
+            )
+            .map(|x| x.1)
+            .map_err(|e| {
+                tracing::error!("Error building oauth slack client: {e}");
+                e
+            })
+            .ok()
+        })
+        .flatten();
+    let all_clients = AllClients { logins, connects, slack };
+    tracing::info!("Final oauth config: {all_clients:#?}");
+    Ok(all_clients)
 }
 
+use anyhow::anyhow;
 pub fn build_basic_client(
     name: String,
     config: OAuthConfig,
@@ -256,9 +297,11 @@ pub fn build_basic_client(
     login: bool,
     base_url: &str,
     override_callback: Option<String>,
-) -> (String, OClient) {
-    let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
-    let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL");
+) -> error::Result<(String, OClient)> {
+    let auth_url = Url::parse(&config.auth_url)
+        .map_err(|e| anyhow!("Invalid authorization endpoint URL: {e}"))?;
+    let token_url =
+        Url::parse(&config.token_url).map_err(|e| anyhow!("Invalid token endpoint URL: {e}"))?;
 
     let redirect_url = if login {
         format!("{base_url}/user/login_callback/{name}")
@@ -273,9 +316,12 @@ pub fn build_basic_client(
         client.set_auth_type(AuthType::RequestBody);
     }
     client.set_client_secret(client_params.secret.clone());
-    client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL"));
+    client.set_redirect_url(
+        Url::parse(&redirect_url).map_err(|e| anyhow!("Invalid redirect URL: {e}"))?,
+    );
+
     // Set up the config for the Github OAuth2 process.
-    (name.to_string(), client)
+    Ok((name.to_string(), client))
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -295,6 +341,8 @@ pub struct SlackTokenResponse {
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct TokenResponse {
     access_token: AccessToken,
+    #[serde(deserialize_with = "maybe_number_opt")]
+    #[serde(default)]
     expires_in: Option<u64>,
     refresh_token: Option<RefreshToken>,
     #[serde(deserialize_with = "helpers::deserialize_space_delimited_vec")]
@@ -314,7 +362,7 @@ async fn connect(
     cookies: Cookies,
 ) -> error::Result<Redirect> {
     let mut query = query.clone();
-    let connects = &OAUTH_CLIENTS.connects;
+    let connects = &OAUTH_CLIENTS.read().await.connects;
     let scopes = query
         .get("scopes")
         .map(|x| x.split('+').map(|x| x.to_owned()).collect());
@@ -330,7 +378,7 @@ async fn connect(
         cookies,
         scopes,
         extra_params,
-        *IS_SECURE,
+        IS_SECURE.read().await.clone(),
     )
 }
 
@@ -341,6 +389,7 @@ struct CreateAccount {
     refresh_token: Option<String>,
     expires_in: i64,
 }
+
 async fn create_account(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -405,6 +454,8 @@ struct Logins {
 async fn list_logins(Extension(sso): Extension<Arc<SamlSsoLogin>>) -> error::JsonResult<Logins> {
     Ok(Json(Logins {
         oauth: OAUTH_CLIENTS
+            .read()
+            .await
             .logins
             .keys()
             .map(|x| x.to_owned())
@@ -420,7 +471,7 @@ struct ScopesAndParams {
 }
 async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
     Ok(Json(
-        (&OAUTH_CLIENTS.connects)
+        (&OAUTH_CLIENTS.read().await.connects)
             .into_iter()
             .map(|(k, v)| {
                 (
@@ -437,6 +488,8 @@ async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>>
 
 async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     let mut client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -452,7 +505,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     client.add_scope("commands");
     let url = client.authorize_url(&state);
 
-    set_cookie(&state, cookies, *IS_SECURE);
+    set_cookie(&state, cookies, IS_SECURE.read().await.clone());
     Ok(Redirect::to(url.as_str()))
 }
 
@@ -495,8 +548,15 @@ async fn disconnect_slack(
 }
 
 async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
-    let clients = &OAUTH_CLIENTS.logins;
-    oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
+    let clients = &OAUTH_CLIENTS.read().await.logins;
+    oauth_redirect(
+        clients,
+        client_name,
+        cookies,
+        None,
+        None,
+        IS_SECURE.read().await.clone(),
+    )
 }
 
 #[derive(Deserialize)]
@@ -531,6 +591,8 @@ pub async fn _refresh_token<'c>(
     .await?;
     let account = not_found_if_none(account, "Account", &id.to_string())?;
     let client = (&OAUTH_CLIENTS
+        .read()
+        .await
         .connects
         .get(&account.client)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
@@ -597,7 +659,7 @@ pub async fn _refresh_token<'c>(
 
 async fn _exchange_token(client: OClient, refresh_token: &str) -> Result<TokenResponse, Error> {
     let token_json = client
-        .exchange_refresh_token(&RefreshToken::from(refresh_token.clone()))
+        .exchange_refresh_token(&RefreshToken::from(refresh_token))
         .with_client(&HTTP_CLIENT)
         .execute::<serde_json::Value>()
         .await
@@ -621,11 +683,10 @@ async fn connect_callback(
     Path(client_name): Path<String>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::JsonResult<TokenResponse> {
-    let client_w_scopes = OAUTH_CLIENTS
-        .connects
+    let connects = &OAUTH_CLIENTS.read().await.connects;
+    let client_w_scopes = connects
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
-
     let client = client_w_scopes.client.to_owned();
     let extra_params = client_w_scopes.extra_params_callback.clone();
     let token_response =
@@ -643,6 +704,8 @@ async fn connect_slack_callback(
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
     let client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -857,7 +920,7 @@ async fn slack_command(
                 None,
             )
             .await?;
-            let url = BASE_URL.to_owned();
+            let url = BASE_URL.read().await.clone();
             tx.commit().await?;
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
@@ -879,10 +942,13 @@ async fn login_callback(
     Extension(webhook): Extension<WebhookShared>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
-    let client_w_config = &OAUTH_CLIENTS
-        .logins
-        .get(&client_name)
-        .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
+    let client_w_config = {
+        let clients = OAUTH_CLIENTS.read().await.logins.clone();
+        clients
+            .get(&client_name)
+            .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
+            .clone()
+    };
     let client = client_w_config.client.to_owned();
     let token_res =
         exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;

backend/windmill-api/src/resources.rs

@@ -8,7 +8,7 @@
 
 use crate::{
     db::{ApiAuthed, DB},
-    users::{maybe_refresh_folders, require_owner_of_path},
+    users::{maybe_refresh_folders, require_owner_of_path, Tokened},
     webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -21,16 +21,20 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{FromRow, Postgres, Transaction};
+use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     db::UserDB,
     error::{Error, JsonResult, Result},
+    jobs::QueuedJob,
     utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    variables,
 };
 
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
+        .route("/list_search", get(list_search_resources))
         .route("/list_names/:type", get(list_names))
         .route("/get/*path", get(get_resource))
         .route("/exists/*path", get(exists_resource))
@@ -144,6 +148,37 @@ async fn list_names(
     Ok(Json(rows))
 }
 
+#[derive(Serialize, FromRow)]
+pub struct SearchResource {
+    path: String,
+    value: serde_json::Value,
+}
+async fn list_search_resources(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<SearchResource>> {
+    let mut tx = user_db.begin(&authed).await?;
+    #[cfg(feature = "enterprise")]
+    let n = 1000;
+
+    #[cfg(not(feature = "enterprise"))]
+    let n = 3;
+
+    let rows = sqlx::query_as!(
+        SearchResource,
+        "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2",
+        &w_id,
+        n
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_resources(
     authed: ApiAuthed,
     Query(lq): Query<ListResourceQuery>,
@@ -271,10 +306,16 @@ async fn get_resource_value(
     Ok(Json(value))
 }
 
+#[derive(Deserialize)]
+struct JobInfo {
+    job_id: Option<Uuid>,
+}
 async fn get_resource_value_interpolated(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Tokened { token }: Tokened,
     Path((w_id, path)): Path<(String, StripPath)>,
+    Query(job_info): Query<JobInfo>,
 ) -> JsonResult<Option<serde_json::Value>> {
     let path = path.to_path();
     let mut tx = user_db.clone().begin(&authed).await?;
@@ -291,7 +332,7 @@ async fn get_resource_value_interpolated(
     let value = not_found_if_none(value_o, "Resource", path)?;
     if let Some(value) = value {
         Ok(Json(Some(
-            transform_json_value(&authed, &user_db, &w_id, value).await?,
+            transform_json_value(&authed, &user_db, &w_id, value, &job_info.job_id, &token).await?,
         )))
     } else {
         Ok(Json(None))
@@ -306,6 +347,8 @@ pub async fn transform_json_value<'c>(
     user_db: &UserDB,
     workspace: &str,
     v: Value,
+    job_id: &Option<Uuid>,
+    token: &str,
 ) -> Result<Value> {
     match v {
         Value::String(y) if y.starts_with("$var:") => {
@@ -331,16 +374,65 @@ pub async fn transform_json_value<'c>(
             tx.commit().await?;
             let v = not_found_if_none(v, "Resource", path)?;
             if let Some(v) = v {
-                transform_json_value(authed, user_db, workspace, v).await
+                transform_json_value(authed, user_db, workspace, v, job_id, token).await
             } else {
                 Ok(Value::Null)
             }
         }
+        Value::String(y) if y.starts_with("$") && job_id.is_some() => {
+            let mut tx = user_db.clone().begin(authed).await?;
+            let job = sqlx::query_as::<_, QueuedJob>(
+                "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2",
+            )
+            .bind(job_id.unwrap())
+            .bind(workspace)
+            .fetch_optional(&mut *tx)
+            .await?;
+            tx.commit().await?;
+
+            let job = not_found_if_none(job, "Job", job_id.unwrap().to_string())?;
+
+            let flow_path = if let Some(uuid) = job.parent_job {
+                let mut tx: Transaction<'_, Postgres> = user_db.clone().begin(authed).await?;
+                let p = sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid)
+                    .fetch_optional(&mut *tx)
+                    .await?
+                    .flatten();
+                tx.commit().await?;
+                p
+            } else {
+                None
+            };
+
+            let variables = variables::get_reserved_variables(
+                &job.workspace_id,
+                token,
+                &job.email,
+                &job.created_by,
+                &job.id.to_string(),
+                &job.permissioned_as,
+                job.script_path.clone(),
+                job.parent_job.map(|x| x.to_string()),
+                flow_path,
+                job.schedule_path.clone(),
+                job.flow_step_id.clone(),
+            )
+            .await;
+
+            let name = y.strip_prefix("$").unwrap();
+
+            let value = variables
+                .iter()
+                .find(|x| x.name == name)
+                .map(|x| x.value.clone())
+                .unwrap_or_else(|| y);
+            Ok(serde_json::json!(value))
+        }
         Value::Object(mut m) => {
             for (a, b) in m.clone().into_iter() {
                 m.insert(
                     a.clone(),
-                    transform_json_value(authed, user_db, workspace, b).await?,
+                    transform_json_value(authed, user_db, workspace, b, job_id, token).await?,
                 );
             }
             Ok(Value::Object(m))