Merge branch 'main' into instructions_helm

* chore(main): release 1.177.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -1,19 +1,12 @@
-DB_PASSWORD=changeme
 DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
 
 WM_IMAGE=ghcr.io/windmill-labs/windmill:main
 WM_LICENSE_KEY=""
 
 # For Enterprise Edition, comment the 2 lines above and uncomment below
-# WINDMILL_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
+# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
 # WM_LICENSE_KEY="<id>.<expiry>.<signature>"
 
 
-# this is the url that your instance is publicly exposed to
-WM_BASE_URL=http://localhost
-
 # To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
 # To have caddy take care of automatic TLS
-
-
-WM_REQUEST_SIZE_LIMIT=50097152 # 50MB (The size limit for any requests, including script/flow args inputs)

.github/DockerfileBackendTests

@@ -58,7 +58,7 @@ RUN /usr/local/bin/python3 -m pip install nltk
 RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
-
+COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
 COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
 RUN apt-get update \

.github/change-versions.sh

@@ -5,7 +5,7 @@ echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json

.github/workflows/backend-test.yml

@@ -42,4 +42,4 @@ jobs:
           mkdir frontend/build && cd backend && touch
           windmill-api/openapi-deref.yaml &&
           DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
-          --features enterprise test --all -- --nocapture
+          test --features enterprise --all -- --nocapture 

.github/workflows/benchmark.yml

@@ -19,15 +19,28 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s
           --health-retries 5
       windmill:
-        image: ghcr.io/windmill-labs/windmill:main
+        image: ghcr.io/windmill-labs/windmill-ee:main
         env:
           DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow
         options: >-
           --pull always --health-interval 10s --health-timeout 5s
           --health-retries 5 --health-cmd "curl
           http://localhost:8000/api/version"
         ports:
           - 8000:8000
+      windmill-worker:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          DISABLE_SERVER: true
+          WORKER_GROUP: dedicated
+          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+        options: >-
+          --pull always
     steps:
       - uses: denoland/setup-deno@v1
         with:
@@ -37,14 +50,9 @@ jobs:
           ref: benchmarks
       - name: benchmark
         timeout-minutes: 10
-        run: deno run --unstable -A
+        run: deno run --unstable -A -r
           https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-          --host http://localhost:8000
-          -r
-          -e admin@windmill.dev
-          -p changeme
           -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
-          --branch ${GITHUB_REF##ref/head/}
       - name: Push changes
         run: |
           pwd

CHANGELOG.md

@@ -1,6 +1,81 @@
 # Changelog
 
 
+## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
+
+
+### Features
+
+* add custom oauth support ([#2336](https://github.com/windmill-labs/windmill/issues/2336)) ([01277f4](https://github.com/windmill-labs/windmill/commit/01277f4d3b8bb04b955d5bbb2ed69c1c7c8f4f9e))
+* support automatic reconnection to pg ([ccaa05d](https://github.com/windmill-labs/windmill/commit/ccaa05d4bf5954c3fb8678239d2962cac6550a5a))
+
+
+### Bug Fixes
+
+* fix resource type picker object reinitialization ([f0f15c4](https://github.com/windmill-labs/windmill/commit/f0f15c47cb35cc1e3cfa13549465803a1e970770))
+* **frontend:** Fix build ([#2330](https://github.com/windmill-labs/windmill/issues/2330)) ([46592af](https://github.com/windmill-labs/windmill/commit/46592affd3d51b54632a2a7a281c11141edcb4a5))
+* **frontend:** Fix markdown dark mode ([#2329](https://github.com/windmill-labs/windmill/issues/2329)) ([6c19740](https://github.com/windmill-labs/windmill/commit/6c197407185810f43c47d4107007bd69814a1d65))
+* set min size of components to 1 ([d298093](https://github.com/windmill-labs/windmill/commit/d298093e29bd9983c7631a8f8c80e47b768bb93c))
+
+## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24)
+
+
+### Features
+
+* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254))
+* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6))
+* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33))
+* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85))
+* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8))
+
+
+### Bug Fixes
+
+* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0))
+* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64))
+* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1))
+* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9))
+* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781))
+* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55))
+* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9))
+* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91))
+
+## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19)
+
+
+### Features
+
+* add batch jobs ([#2306](https://github.com/windmill-labs/windmill/issues/2306)) ([5867e5d](https://github.com/windmill-labs/windmill/commit/5867e5d0f80fd515fab165659831b5ee9a8c3f97))
+* add dediacted worker env var ([#2296](https://github.com/windmill-labs/windmill/issues/2296)) ([e0c6eee](https://github.com/windmill-labs/windmill/commit/e0c6eee16e535b3a7d803a7978e463404f5fec30))
+* dedicated benchmarks ([#2297](https://github.com/windmill-labs/windmill/issues/2297)) ([c549239](https://github.com/windmill-labs/windmill/commit/c5492396843ddd9143ffe890696d0317c970de36))
+* **frontend:** Add component control doc ([#2295](https://github.com/windmill-labs/windmill/issues/2295)) ([26f8863](https://github.com/windmill-labs/windmill/commit/26f88636f0b972d4fe4931ed02135c38b27a56d2))
+* suggest adding openai key on workspace creation ([a6b3b2f](https://github.com/windmill-labs/windmill/commit/a6b3b2f63b317825a3d80218cbb606b9f610c221))
+* support pinned versions for bun in deployed scripts ([03806dc](https://github.com/windmill-labs/windmill/commit/03806dc3907cba724be14acb6aadf5be6e35cdb6))
+
+
+### Bug Fixes
+
+* add HOME to bun and deno ([0e3ecc7](https://github.com/windmill-labs/windmill/commit/0e3ecc7d6025c173135f20bacc33a0dc972ec222))
+* add queue_count to metrics ([9ced883](https://github.com/windmill-labs/windmill/commit/9ced8834a45151c6900b1eb33eca2cff4886a065))
+* ai improve prompts ([#2310](https://github.com/windmill-labs/windmill/issues/2310)) ([b647213](https://github.com/windmill-labs/windmill/commit/b647213b2c968b0cb1f90c97d94e8023c415dd55))
+* **frontend:** add missing key ([#2299](https://github.com/windmill-labs/windmill/issues/2299)) ([39d2467](https://github.com/windmill-labs/windmill/commit/39d24672ddd696372e55e9b4566f322a322385a8))
+* **frontend:** Always mount components ([#2309](https://github.com/windmill-labs/windmill/issues/2309)) ([34f94aa](https://github.com/windmill-labs/windmill/commit/34f94aa50e92254114c046fa8b7e900d93807937))
+* **frontend:** fix alignment ([#2307](https://github.com/windmill-labs/windmill/issues/2307)) ([f9fc6f1](https://github.com/windmill-labs/windmill/commit/f9fc6f19482e68c9ccba0014879fd8761662c36a))
+* **frontend:** Fix rich result styling + add title and hideDetails config ([#2294](https://github.com/windmill-labs/windmill/issues/2294)) ([732daef](https://github.com/windmill-labs/windmill/commit/732daef1c3515f7df3e09deac691bb585f9859cd))
+* **frontend:** fix tab styling + component bg ([#2308](https://github.com/windmill-labs/windmill/issues/2308)) ([5e773d3](https://github.com/windmill-labs/windmill/commit/5e773d386343f003425173207c166e3c4eeef956))
+* **frontend:** fix theme make default ([#2304](https://github.com/windmill-labs/windmill/issues/2304)) ([4629819](https://github.com/windmill-labs/windmill/commit/46298197c5333a81b9b8a004027ab9a856bdada4))
+* **frontend:** fix theme UI ([#2305](https://github.com/windmill-labs/windmill/issues/2305)) ([576f76b](https://github.com/windmill-labs/windmill/commit/576f76b1ffe9c50c8ccaca8c5e34d0ec03aebf3f))
+* validate more strongly usernames ([47094bb](https://github.com/windmill-labs/windmill/commit/47094bb8d1c6f4ba621d42515dede061fd04afdd))
+
+## [1.174.0](https://github.com/windmill-labs/windmill/compare/v1.173.0...v1.174.0) (2023-09-15)
+
+
+### Features
+
+* ai gen support all langs ([#2276](https://github.com/windmill-labs/windmill/issues/2276)) ([39590b3](https://github.com/windmill-labs/windmill/commit/39590b3d2592b2d08117c0f70829c13f1efb4885))
+* bun absolute/relative imports + tests ([#2286](https://github.com/windmill-labs/windmill/issues/2286)) ([e5ce85b](https://github.com/windmill-labs/windmill/commit/e5ce85b9affe665342f24b1d39ce3d03db09b941))
+* **frontend:** Global CSS editor ([#2178](https://github.com/windmill-labs/windmill/issues/2178)) ([7e9ee39](https://github.com/windmill-labs/windmill/commit/7e9ee39aa69bc31766b5e4f4aab498c8f14067cd))
+
 ## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
 
 

Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl nodejs npm
+    curl nodejs
 
 RUN rustup component add rustfmt
 
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.2 /usr/local/bin/bun /usr/bin/bun
 
 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/

LICENSE

@@ -1,17 +1,22 @@
-
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
+./LICENSE-AGPL)
 
-Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
-Every file is under License AGPL unless otherwise specified 
-or belonging to one of the below cases:
+Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
+specified. Every file is under License AGPL unless otherwise specified or
+belonging to one of the below cases:
 
-The files under backend/ are AGPLv3 Licensed, except any snippets of code under the compile flag "enterprise". Those snippets and files are under a proprietary and commerccial license.
-The files under frontend/ are AGPLv3 Licensed, except any snippers of code that require a positive license check to be activated. Those snippets and files are under a proprietary and commercial license.
-Private and public forks MUST not include any of the above proprietary and commercial code. Windmill Labs, Inc. provide tools to clean the codebase from those snippets upon demand.
-The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
+The files under backend/ are AGPLv3 Licensed, except any snippets of code under
+the compile flag "enterprise". Those snippets and files are under a proprietary
+and commercial license. The files under frontend/ are AGPLv3 Licensed, except
+any snippets of code that require a positive license check to be activated.
+Those snippets and files are under a proprietary and commercial license. Private
+and public forks MUST not include any of the above proprietary and commercial
+code. Windmill Labs, Inc. provide tools to clean the codebase from those
+snippets upon demand. The files under python-client/ deno-client/ go-client/ are
+Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
 
-All third party components incorporated into the Windmill Software are licensed under the
-original license provided by the owner of the applicable component.
+All third party components incorporated into the Windmill Software are licensed
+under the original license provided by the owner of the applicable component.

README.md

@@ -211,7 +211,6 @@ compiling from source or using without a postgres super user, see
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
 curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/oauth.json -o oauth.json
 
 docker compose up -d
 ```
@@ -377,6 +376,7 @@ it being synced automatically everyday.
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path>                                                                                                            | Worker                |
 | CUSTOM_TAGS                                   | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
 | JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
 | WAIT_RESULT_FAST_POLL_INTERVAL_MS             | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
@@ -395,7 +395,7 @@ it being synced automatically everyday.
 | SMTP_USERNAME                                 | None                                       | username for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_PASSWORD                                 | None                                       | password for the smtp server to send invite emails                                                                                                                                                 | Server                |
 | SMTP_TLS_IMPLICIT                             | false                                      | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails                                                                                                 | Server                |
-| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | false                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
+| CREATE_WORKSPACE_REQUIRE_SUPERADMIN           | true                                      | If true, only superadmin can create workspaces                                                                                                                                                     | Server                |
 | GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None                                       | Path to a script to run when a root job fails. The script will be run in and from the admins workspace                                                                                             | Server                |
 | WHITELIST_ENVS                                | None                                       | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers                                                                                           | Worker                |
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |

backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "DELETE FROM worker_group_config WHERE name = $1 RETURNING name",
+  "query": "DELETE FROM config WHERE name = $1 RETURNING name",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       false
     ]
   },
-  "hash": "210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174"
+  "hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a"
 }

backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json

@@ -37,7 +37,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json

@@ -67,7 +67,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, count(*) as count FROM queue WHERE\n        scheduled_for <= now() - ('3 seconds')::interval AND running = false\n        GROUP BY tag",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a"
+}

backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json

@@ -28,7 +28,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a"
+}

backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json

@@ -0,0 +1,73 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH uuid_table as (\n            select gen_random_uuid() as uuid from generate_series(1, $11)\n        )\n        INSERT INTO queue \n            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n            (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n        RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d"
+}

backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT config FROM config WHERE name = 'server'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+}

backend/.sqlx/query-8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM resource_type WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6"
+}

backend/.sqlx/query-903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56"
-}

backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json

@@ -60,7 +60,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49"
+}

backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json

@@ -42,7 +42,6 @@
                 "bash",
                 "postgresql",
                 "nativets",
-                "Nativets",
                 "bun",
                 "mysql",
                 "bigquery",

backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT config FROM worker_group_config WHERE name = $1",
+  "query": "SELECT config FROM config WHERE name = $1",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       true
     ]
   },
-  "hash": "6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c"
+  "hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429"
 }

backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT * FROM worker_group_config",
+  "query": "SELECT * FROM config WHERE name LIKE 'worker__%'",
   "describe": {
     "columns": [
       {
@@ -22,5 +22,5 @@
       true
     ]
   },
-  "hash": "7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987"
+  "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7"
 }

backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9"
+}

backend/.sqlx/query-d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4"
+}

backend/.sqlx/query-ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hash FROM script WHERE path = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
+}

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.173.0"
+version = "1.177.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.173.0"
+version = "1.177.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -54,7 +54,6 @@ tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
 git-version.workspace = true
-rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
@@ -64,6 +63,8 @@ once_cell.workspace = true
 prometheus.workspace = true
 uuid.workspace = true
 gethostname.workspace = true
+serde_json.workspace = true
+serde.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
@@ -112,7 +113,7 @@ mime_guess = "^2"
 hex = "^0"
 sql-builder = "^3"
 argon2 = "^0"
-retainer = "^0"
+quick_cache = "^0"
 rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"

backend/migrations/20230915084031_add_themes.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230915084031_add_themes.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;
+INSERT INTO resource (workspace_id, path, value, description, resource_type) SELECT id, 'f/app_themes/theme_0', '{"name": "Default Theme", "value": ""}', 'The default app theme', 'app_theme' FROM workspace ON CONFLICT DO NOTHING

backend/migrations/20230917075933_rename_config.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE IF exists config RENAME TO worker_group_config ;

backend/migrations/20230917075933_rename_config.up.sql

@@ -0,0 +1,29 @@
+-- Add up migration script here
+ALTER TABLE IF exists worker_group_config RENAME TO config;
+UPDATE config SET name = 'worker__' || name;
+
+CREATE FUNCTION "notify_config_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_config_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_config_change"
+ AFTER INSERT OR UPDATE ON "config"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_config_change" ();
+
+CREATE FUNCTION "notify_global_setting_change" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_global_setting_change"
+ AFTER INSERT OR UPDATE ON "global_settings"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_global_setting_change" ();

backend/migrations/20230921084441_add_default_worker_groups.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230921084441_add_default_worker_groups.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+INSERT INTO config (name, config) VALUES  
+    ('worker__default', '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun"]}'), 
+    ('worker__native', '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql", "snowflake"]}') ON CONFLICT DO NOTHING;

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -35,7 +35,9 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "dateutil" => "python-dateutil",
     "mailparser" => "mail-parser",
     "mailparser-reply" => "mail-parser-reply",
-    "gitlab" => "python-gitlab"
+    "gitlab" => "python-gitlab",
+    "smbclient" => "smbprotocol",
+    "playhouse" => "peewee",
 };
 
 fn replace_import(x: String) -> String {

backend/src/ee.rs

@@ -1,41 +1,49 @@
 #[cfg(feature = "enterprise")]
-use base64::Engine;
-#[cfg(feature = "enterprise")]
-use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
-#[cfg(feature = "enterprise")]
-use sha2::Sha256;
+use windmill_common::error;
 
-#[cfg(feature = "enterprise")]
-pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
-    if let Some(license_key) = license_key {
-        let mut splitted_lk = license_key.split(".");
-        if splitted_lk.clone().count() != 3 {
-            panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
-        }
-        let id = splitted_lk.next().unwrap();
-        let expiry = splitted_lk.next().unwrap();
-        let signature_b64 = splitted_lk.next().unwrap();
+pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
+    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
 
-        let expiry_nb = expiry.parse::<u64>()?;
-        if expiry_nb < chrono::Utc::now().timestamp() as u64 {
-            panic!(
-                "License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
-                chrono::Utc::now().timestamp()
-            );
-        }
-        const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
-        let pub_key = rsa::RsaPublicKey::from_public_key_der(
-            &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
-        )?;
-        let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
-        rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
-            .verify(
-                &format!("{id}{expiry}").as_bytes(),
-                &rsa::pss::Signature::from(signature),
-            )
-            .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
-    } else {
-        panic!("License key is required for the enterprise edition");
+    let id = validate_license_key(license_key.clone()).await?;
+    {
+        let mut l = LICENSE_KEY_ID.write().await;
+        *l = id.to_string()
     }
+
+    {
+        let mut l = LICENSE_KEY.write().await;
+        *l = license_key
+    }
+    {
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = true
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn verify_license_key() -> error::Result<()> {
+    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::error::to_anyhow;
+
+    let expiry_nb = LICENSE_KEY
+        .read()
+        .await
+        .clone()
+        .split(".")
+        .nth(1)
+        .unwrap_or_else(|| "")
+        .parse::<u64>()
+        .map_err(to_anyhow)?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        let mut l = LICENSE_KEY_VALID.write().await;
+        *l = false;
+    };
     Ok(())
 }

backend/src/main.rs

@@ -8,7 +8,8 @@
 
 use gethostname::gethostname;
 use git_version::git_version;
-use sqlx::{Pool, Postgres};
+use rand::Rng;
+use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     sync::Arc,
@@ -18,8 +19,15 @@ use tokio::{
     fs::{metadata, DirBuilder},
     sync::RwLock,
 };
-use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
+use windmill_common::{
+    global_settings::{
+        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, LICENSE_KEY_SETTING, OAUTH_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
+    },
+    utils::rd_string,
+    worker::{reload_custom_tags_setting, WORKER_GROUP},
+    DB, METRICS_ADDR,
+};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -27,7 +35,10 @@ use windmill_worker::{
     PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
 
-use crate::monitor::monitor_db;
+use crate::monitor::{
+    initial_load, monitor_db, reload_base_url_setting, reload_license_key,
+    reload_retention_period_setting, reload_server_config, reload_worker_config,
+};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 1;
@@ -127,16 +138,6 @@ Windmill Community Edition {GIT_VERSION}
 
     display_config(&ENV_SETTINGS);
 
-    tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
-    if let Some(ref smtp) = *SMTP_CLIENT {
-        tracing::info!("Smtp client defined. Testing connection...");
-        if let Err(e) = smtp.connect().await {
-            tracing::error!("Failed to connect to smtp server: {}", e);
-        } else {
-            tracing::info!("Smtp client connected.");
-        }
-    }
-
     let worker_mode = num_workers > 0;
 
     if server_mode || worker_mode {
@@ -151,15 +152,9 @@ Windmill Community Edition {GIT_VERSION}
         // since it's only on server mode, the port is statically defined
         let base_internal_url: String = format!("http://localhost:{}", port.to_string());
 
-        monitor_db(
-            &db,
-            tx.clone(),
-            &base_internal_url,
-            rsmq.clone(),
-            worker_mode,
-            server_mode,
-        )
-        .await;
+        initial_load(&db, tx.clone(), worker_mode, server_mode).await;
+
+        monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
 
         if std::env::var("BASE_INTERNAL_URL").is_ok() {
             tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
@@ -201,21 +196,103 @@ Windmill Community Edition {GIT_VERSION}
 
             let mut rx = rx.resubscribe();
             let base_internal_url = base_internal_url.to_string();
+            let rd_delay = rand::thread_rng().gen_range(0..30);
             tokio::spawn(async move {
                 //monitor_db is applied at start, no need to apply it twice
-                tokio::time::sleep(Duration::from_secs(30)).await;
+                tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+
+                let mut listener = retry_listen_pg(&db).await;
+
                 loop {
-                    monitor_db(
-                        &db,
-                        tx.clone(),
-                        &base_internal_url,
-                        rsmq.clone(),
-                        worker_mode,
-                        server_mode,
-                    )
-                    .await;
                     tokio::select! {
-                        _ = tokio::time::sleep(Duration::from_secs(30))    => (),
+                        _ = tokio::time::sleep(Duration::from_secs(30))    => {
+                            monitor_db(
+                                &db,
+                                &base_internal_url,
+                                rsmq.clone(),
+                                server_mode,
+                            )
+                            .await;
+                        },
+                        notification = listener.recv() => {
+                            match notification {
+                                Ok(n) => {
+                                    tracing::info!("Received new pg notification: {n:?}");
+                                    match n.channel() {
+                                        "notify_config_change" => {
+                                            tracing::info!("Config change detected");
+                                            match n.payload() {
+                                                "server" if server_mode => {
+                                                    tracing::info!("Server config change detected");
+                                                    reload_server_config(&db).await;
+                                                },
+                                                a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
+                                                    tracing::info!("Worker config change detected");
+                                                    reload_worker_config(&db, tx.clone(), true).await;
+                                                },
+                                                _ => {
+                                                    ()
+                                                }
+                                            }
+                                        },
+                                        "notify_global_setting_change" => {
+                                            tracing::info!("Global setting change detected");
+                                            match n.payload() {
+                                                BASE_URL_SETTING => {
+                                                    tracing::info!("Base URL setting change detected");
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload base url setting");
+                                                    }
+                                                },
+                                                OAUTH_SETTING => {
+                                                    tracing::info!("OAuth setting change detected");
+                                                    if let Err(e) = reload_base_url_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload oauth setting");
+                                                    }
+                                                },
+                                                CUSTOM_TAGS_SETTING => {
+                                                    tracing::info!("Custom tags setting change detected");
+                                                    if let Err(e) = reload_custom_tags_setting(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload custom tags setting");
+                                                    }
+                                                },
+                                                LICENSE_KEY_SETTING => {
+                                                    tracing::info!("License Key setting change detected");
+                                                    if let Err(e) = reload_license_key(&db).await {
+                                                        tracing::error!(error = %e, "Could not reload license key setting");
+                                                    }
+                                                },
+                                                RETENTION_PERIOD_SECS_SETTING => {
+                                                    tracing::info!("Retention period setting change detected");
+                                                    reload_retention_period_setting(&db).await
+                                                },
+                                                REQUEST_SIZE_LIMIT_SETTING => {
+                                                    tracing::info!("Request limit size change detected, killing server expecting to be restarted");
+                                                    // we wait a bit randomly to avoid having all servers shutdown at same time
+                                                    let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                    if let Err(e) = tx.send(()) {
+                                                        tracing::error!(error = %e, "Could not send killpill to server");
+                                                    }
+                                                }
+                                                a @_ => {
+                                                    tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
+                                                }
+                                            }
+                                        },
+                                        _ => {
+                                            tracing::warn!("Unknown notification received");
+                                            continue;
+                                        }
+                                    }
+                                },
+                                Err(e) => {
+                                    tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
+                                    listener = retry_listen_pg(&db).await;
+                                    continue;
+                                }
+                            };
+                        },
                         _ = rx.recv() => {
                                 println!("received killpill for monitor job");
                                 break;
@@ -245,6 +322,40 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
+async fn listen_pg(db: &DB) -> Option<PgListener> {
+    let mut listener = match PgListener::connect_with(&db).await {
+        Ok(l) => l,
+        Err(e) => {
+            tracing::error!(error = %e, "Could not connect to database");
+            return None;
+        }
+    };
+
+    if let Err(e) = listener
+        .listen_all(vec!["notify_config_change", "notify_global_setting_change"])
+        .await
+    {
+        tracing::error!(error = %e, "Could not listen to database");
+        return None;
+    }
+
+    return Some(listener);
+}
+
+async fn retry_listen_pg(db: &DB) -> PgListener {
+    let mut listener = listen_pg(db).await;
+    loop {
+        if listener.is_none() {
+            tracing::info!("Retrying listening to pg listen in 5 seconds");
+            tokio::time::sleep(Duration::from_secs(5)).await;
+            listener = listen_pg(db).await;
+        } else {
+            tracing::info!("Successfully connected to pg listen");
+            return listener.unwrap();
+        }
+    }
+}
+
 fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
@@ -270,14 +381,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
     base_internal_url: String,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
-    #[cfg(feature = "enterprise")]
-    ee::verify_license_key(LICENSE_KEY.clone())?;
-
-    #[cfg(not(feature = "enterprise"))]
-    if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) {
-        panic!("License key is required ONLY for the enterprise edition");
-    }
-
     let instance_name = gethostname()
         .to_str()
         .map(|x| {

backend/src/monitor.rs

@@ -1,17 +1,41 @@
+use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc};
+
 use once_cell::sync::OnceCell;
+use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
-use tokio::{join, sync::mpsc};
+use tokio::{
+    join,
+    sync::{mpsc, RwLock},
+};
 use uuid::Uuid;
+use windmill_api::{
+    oauth2::{build_oauth_clients, OAuthClient},
+    DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT,
+};
 use windmill_common::{
     error,
+    global_settings::{
+        BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        RETENTION_PERIOD_SECS_SETTING,
+    },
     jobs::{JobKind, QueuedJob},
-    worker::{load_worker_config, reload_custom_tags_setting, WORKER_CONFIG},
-    METRICS_ENABLED,
+    server::load_server_config,
+    users::truncate_token,
+    worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
+    BASE_URL, DB, METRICS_ENABLED,
 };
 use windmill_worker::{
     create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
 };
 
+#[cfg(feature = "enterprise")]
+use crate::ee::verify_license_key;
+
+#[cfg(feature = "enterprise")]
+use windmill_api::LICENSE_KEY_VALID;
+
+use crate::ee::set_license_key;
+
 lazy_static::lazy_static! {
     static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
     .ok()
@@ -34,29 +58,26 @@ lazy_static::lazy_static! {
         "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
     )
     .unwrap();
+
+    static ref QUEUE_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
+        "queue_count",
+        "Number of jobs in the queue",
+        &["tag"]
+    ).unwrap();
+
+    static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
+
 }
 
-pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+pub async fn initial_load(
     db: &Pool<Postgres>,
     tx: tokio::sync::broadcast::Sender<()>,
-    base_internal_url: &str,
-    rsmq: Option<R>,
     worker_mode: bool,
     server_mode: bool,
 ) {
-    let zombie_jobs_f = async {
-        if server_mode {
-            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
-        }
-    };
-    let expired_items_f = async {
-        if server_mode {
-            windmill_api::delete_expired_items(&db).await;
-        }
-    };
     let reload_worker_config_f = async {
         if worker_mode {
-            reload_worker_config(&db, tx).await;
+            reload_worker_config(&db, tx, false).await;
         }
     };
     let reload_custom_tags_f = async {
@@ -66,15 +87,299 @@ pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
             }
         }
     };
+
+    let reload_base_url_f = async {
+        
+            if let Err(e) = reload_base_url_setting(db).await {
+                tracing::error!("Error reloading base url: {:?}", e)
+            }
+        
+    };
+
+    let reload_server_config_f = async {
+        if server_mode {
+            reload_server_config(&db).await;
+        }
+    };
+    let reload_retention_period_f = async {
+        if server_mode {
+            reload_retention_period_setting(&db).await;
+        }
+    };
+
+    let reload_request_size_f = async {
+        if server_mode {
+            reload_request_size(&db).await;
+        }
+    };
+
+    let reload_license_key_f = async {
+        if server_mode {
+            #[cfg(feature = "enterprise")]
+            if let Err(e) = reload_license_key(&db).await {
+                tracing::error!("Error reloading license key: {:?}", e)
+            }
+        }
+    };
+
     join!(
-        expired_items_f,
-        zombie_jobs_f,
         reload_worker_config_f,
-        reload_custom_tags_f
+        reload_server_config_f,
+        reload_custom_tags_f,
+        reload_request_size_f,
+        reload_base_url_f,
+        reload_retention_period_f,
+        reload_license_key_f
     );
 }
 
-pub async fn reload_worker_config(db: &Pool<Postgres>, tx: tokio::sync::broadcast::Sender<()>) {
+pub async fn delete_expired_items(db: &DB) -> () {
+    let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
+        "DELETE FROM token WHERE expiration <= now()
+        RETURNING concat(substring(token for 10), '*****')",
+    )
+    .fetch_all(db)
+    .await;
+
+    match tokens_deleted_r {
+        Ok(tokens) => {
+            if tokens.len() > 0 {
+                tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
+    }
+
+    let pip_resolution_r = sqlx::query_scalar!(
+        "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
+    )
+    .fetch_all(db)
+    .await;
+
+    match pip_resolution_r {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
+    }
+
+    let deleted_cache = sqlx::query_scalar!(
+            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
+        )
+        .fetch_all(db)
+        .await;
+
+    match deleted_cache {
+        Ok(res) => {
+            if res.len() > 0 {
+                tracing::info!("deleted {} cache resource: {:?}", res.len(), res)
+            }
+        }
+        Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
+    }
+
+    let job_retention_secs = *JOB_RETENTION_SECS.read().await;
+    if job_retention_secs > 0 {
+        let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                job_retention_secs
+            )
+            .fetch_all(db)
+            .await;
+
+        match deleted_jobs {
+            Ok(deleted_jobs) => {
+                if deleted_jobs.len() > 0 {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
+                        deleted_jobs.len(),
+                        job_retention_secs,
+                        deleted_jobs,
+                    )
+                }
+            }
+            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+        }
+    }
+}
+
+pub async fn reload_retention_period_setting(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        RETENTION_PERIOD_SECS_SETTING,
+        "JOB_RETENTION_SECS",
+        60 * 60 * 24 * 60,
+        JOB_RETENTION_SECS.clone(),
+        |x| x,
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_request_size(db: &DB) {
+    if let Err(e) = reload_setting(
+        db,
+        REQUEST_SIZE_LIMIT_SETTING,
+        "REQUEST_SIZE_LIMIT",
+        DEFAULT_BODY_LIMIT,
+        REQUEST_SIZE_LIMIT.clone(),
+        |x| x.mul(1024 * 1024),
+    )
+    .await
+    {
+        tracing::error!("Error reloading retention period: {:?}", e)
+    }
+}
+
+pub async fn reload_license_key(db: &DB) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        LICENSE_KEY_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var("LICENSE_KEY")
+        .ok()
+        .and_then(|x| x.parse::<String>().ok())
+        .unwrap_or(String::new());
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting LICENSE_KEY from db config: {}",
+                truncate_token(&v)
+            );
+            value = v;
+        } else {
+            tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value);
+        }
+    };
+
+    set_license_key(value).await?;
+
+    Ok(())
+}
+
+pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
+    db: &DB,
+    setting_name: &str,
+    std_env_var: &str,
+    default: T,
+    lock: Arc<RwLock<T>>,
+    transformer: fn(T) -> T,
+) -> error::Result<()> {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        setting_name
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let mut value = std::env::var(std_env_var)
+        .ok()
+        .and_then(|x| x.parse::<T>().ok())
+        .unwrap_or(default);
+
+    if let Some(q) = q {
+        if let Ok(v) = serde_json::from_value::<T>(q.value.clone()) {
+            tracing::info!(
+                "Loaded setting {setting_name} from db config: {:#?}",
+                &q.value
+            );
+            value = transformer(v);
+        } else {
+            tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
+        }
+    };
+
+    {
+        let mut l = lock.write().await;
+        *l = value;
+    }
+
+    Ok(())
+}
+
+pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+    server_mode: bool,
+) {
+    let zombie_jobs_f = async {
+        if server_mode {
+            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+        }
+    };
+    let expired_items_f = async {
+        if server_mode {
+            delete_expired_items(&db).await;
+        }
+    };
+
+    let verify_license_key_f = async {
+        if server_mode {
+            #[cfg(feature = "enterprise")]
+            if let Err(e) = verify_license_key().await {
+                tracing::error!("Error verifying license key: {:?}", e);
+                let mut l = LICENSE_KEY_VALID.write().await;
+                *l = false;
+            }
+        }
+    };
+
+    let expose_queue_metrics_f = async {
+        if *METRICS_ENABLED && server_mode {
+            expose_queue_metrics(&db).await;
+        }
+    };
+    join!(
+        expired_items_f,
+        zombie_jobs_f,
+        expose_queue_metrics_f,
+        verify_license_key_f
+    );
+}
+
+pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
+    let queue_counts = sqlx::query!(
+        "SELECT tag, count(*) as count FROM queue WHERE
+        scheduled_for <= now() - ('3 seconds')::interval AND running = false
+        GROUP BY tag"
+    )
+    .fetch_all(db)
+    .await
+    .ok()
+    .unwrap_or_else(|| vec![]);
+    for q in queue_counts {
+        let count = q.count.unwrap_or(0);
+        let tag = q.tag;
+        let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+        metric.set(count as i64);
+    }
+}
+
+pub async fn reload_server_config(db: &Pool<Postgres>) {
+    let config = load_server_config(&db).await;
+    if let Err(e) = config {
+        tracing::error!("Error reloading server config: {:?}", e)
+    } else {
+        let mut wc = SERVER_CONFIG.write().await;
+        tracing::info!("Reloading server config...");
+        *wc = config.unwrap()
+    }
+}
+
+pub async fn reload_worker_config(
+    db: &DB,
+    tx: tokio::sync::broadcast::Sender<()>,
+    kill_if_change: bool,
+) {
     let config = load_worker_config(&db).await;
     if let Err(e) = config {
         tracing::error!("Error reloading worker config: {:?}", e)
@@ -82,7 +387,7 @@ pub async fn reload_worker_config(db: &Pool<Postgres>, tx: tokio::sync::broadcas
         let wc = WORKER_CONFIG.read().await;
         let config = config.unwrap();
         if *wc != config {
-            if (*wc).dedicated_worker != config.dedicated_worker {
+            if kill_if_change && (*wc).dedicated_worker != config.dedicated_worker {
                 tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
                 let _ = tx.send(());
             }
@@ -95,6 +400,77 @@ pub async fn reload_worker_config(db: &Pool<Postgres>, tx: tokio::sync::broadcas
     }
 }
 
+pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
+    let q_base_url = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        BASE_URL_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let base_url = if let Some(q) = q_base_url {
+        if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse base_url setting as a string, found: {:#?}",
+                &q.value
+            );
+            std::env::var("BASE_URL")
+                .ok()
+                .unwrap_or_else(|| "http://localhost".to_string())
+        }
+    } else {
+        std::env::var("BASE_URL")
+            .ok()
+            .unwrap_or_else(|| "http://localhost".to_string())
+    };
+
+    let q_oauth = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        OAUTH_SETTING
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let oauths = if let Some(q) = q_oauth {
+        if let Ok(v) =
+            serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.value.clone())
+        {
+            v
+        } else {
+            tracing::error!(
+                "Could not parse oauth setting as a json, found: {:#?}",
+                &q.value
+            );
+            None
+        }
+    } else {
+        None
+    };
+
+    let is_secure = base_url.starts_with("https://");
+
+    {
+        let mut l = OAUTH_CLIENTS.write().await;
+        *l = build_oauth_clients(&base_url, oauths)
+        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
+        .unwrap();
+    }
+
+    {
+        let mut l = BASE_URL.write().await;
+        *l = base_url
+    }
+
+    {
+        let mut l = IS_SECURE.write().await;
+        *l = is_secure;
+    }
+
+    Ok(())
+}
+
 async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     db: &Pool<Postgres>,
     base_internal_url: &str,

backend/tests/fixtures/base.sql

@@ -17,59 +17,6 @@ INSERT INTO workspace_settings (workspace_id) VALUES
 
 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main(fail: boolean = true) {
-  if (fail) {
-    throw new Error("Failed")
-  }
-
-  return "OK"
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_script', -28028598712388162, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Error handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
-'',
-'',
-'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
-
-INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
-'test-workspace',
-'system',
-'
-export async function main() {
-  return "Recovery handler";
-}
-',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
-'',
-'',
-'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
-
-INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
-'test-workspace',
-'system',
-'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
-'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
-'',
-'',
-'f/system/failing_flow'
-);
-
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
 

backend/tests/fixtures/relative_bun.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'bun', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'bun', '');

backend/tests/fixtures/relative_deno.sql

@@ -0,0 +1,44 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system/same_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+export function main() {
+  return "f/system_relative/different_folder_script";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+import { main as test1  } from "/f/system/same_folder_script.ts";
+import { main as test2  } from "../system/same_folder_script.ts";
+import { main as test3  } from "/f/system_relative/different_folder_script.ts";
+import { main as test4  } from "./different_folder_script.ts";
+
+export function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'deno', '');

backend/tests/fixtures/relative_python.sql

@@ -0,0 +1,41 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system/same_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system/same_folder_script', -28028598712388162, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+def main():
+    return "f/system_relative/different_folder_script"
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
+
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'test-user',
+'
+from f.system.same_folder_script import main as test1
+from ..system.same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from .different_folder_script import main as test4
+
+def main():
+    return [test1(), test2(), test3(), test4()]
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
+'',
+'',
+'f/system_relative/nested_script', -28028598712388160, 'python3', '');

backend/tests/fixtures/schedule.sql

@@ -0,0 +1,52 @@
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);

backend/tests/worker.rs

@@ -1,19 +1,30 @@
 use std::sync::Arc;
 
+#[cfg(feature = "enterprise")]
 use chrono::Timelike;
 use futures::StreamExt;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
-use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, query};
-use tokio::{
-    sync::RwLock,
-    time::{timeout, Duration},
-};
+use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
+use tokio::sync::RwLock;
+
+#[cfg(feature = "enterprise")]
+use tokio::time::{timeout, Duration};
+
 use windmill_api_client::types::{
-    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+    CreateFlowBody, RawScript
 };
-use windmill_common::worker::{WORKER_CONFIG, load_worker_config};
+
+use sqlx::query;
+
+#[cfg(feature = "enterprise")]
+use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
+
+use windmill_api_client::types::{NewScript, NewScriptLanguage};
+
+use windmill_common::worker::WORKER_CONFIG;
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
@@ -23,6 +34,8 @@ use windmill_common::{
 use windmill_queue::PushIsolationLevel;
 use serde::Serialize;
 
+use std::str::FromStr;
+
 #[derive(Debug, sqlx::FromRow, Serialize)]
 pub struct CompletedJob {
     pub workspace_id: String,
@@ -31,7 +44,7 @@ pub struct CompletedJob {
     pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub started_at: chrono::DateTime<chrono::Utc>,
-    pub duration_ms: i32,
+    pub duration_ms: i64,
     pub success: bool,
     pub script_path: Option<String>,
     pub args: Option<serde_json::Value>,
@@ -126,6 +139,7 @@ impl ApiServer {
     }
 
     async fn close(self) -> anyhow::Result<()> {
+        println!("closing api server");
         let Self { tx, task, .. } = self;
         drop(tx);
         task.await.unwrap()
@@ -866,7 +880,7 @@ impl RunJob {
             args,
             /* user */ "test-user",
             /* email  */ "test@windmill.dev",
-            /* permissioned_as */ "u/admin".to_string(),
+            /* permissioned_as */ "u/test-user".to_string(),
             /* scheduled_for_o */ None,
             /* schedule_path */ None,
             /* parent_job */ None,
@@ -892,7 +906,8 @@ impl RunJob {
         let uuid = self.push(db).await;
         let listener = listen_for_completed_jobs(db).await;
         in_test_worker(db, listener.find(&uuid), port).await;
-        completed_job(uuid, db).await
+        let r = completed_job(uuid, db).await;
+        r
     }
 }
 
@@ -927,13 +942,12 @@ async fn in_test_worker<Fut: std::future::Future>(
     };
 
     /* ensure the worker quits before we return */
-    drop(quit);
+    quit.send(()).expect("send");
 
     let _: () = worker
         .await
         .expect("worker timed out")
         .expect("worker panicked");
-
     res
 }
 
@@ -960,9 +974,10 @@ fn spawn_test_worker(
     let tx2 = tx.clone();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
+        {
         let mut wc = WORKER_CONFIG.write().await;
-        *wc = load_worker_config(&db).await.unwrap();
-        drop(wc);
+        (*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
+        }
         windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
             &db,
             worker_instance,
@@ -1670,7 +1685,6 @@ echo "hello $msg"
     .arg("msg", json!("world"))
     .run_until_complete(&db, port)
     .await;
-
     assert_eq!(job.json_result(), Some(json!("hello world")));
 }
 
@@ -2603,7 +2617,8 @@ async fn test_rust_client(db: Pool<Postgres>) {
 }
 
 
-#[sqlx::test(fixtures("base"))]
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_script_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2736,7 +2751,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
 }
 
 
-#[sqlx::test(fixtures("base"))]
+#[cfg(feature = "enterprise")]
+#[sqlx::test(fixtures("base", "schedule"))]
 async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     initialize_tracing().await;
     let server = ApiServer::start(db.clone()).await;
@@ -2869,3 +2885,203 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
     )
     .await;
 }
+
+
+async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    client.create_script(
+        "test-workspace",
+        &NewScript {
+            language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
+            content: script_content,
+            path: "f/system/test_import".to_string(),
+            concurrent_limit: vec![],
+            concurrency_time_window_s: vec![],
+            cache_ttl: None,
+            dedicated_worker: None,
+            description: "".to_string(),
+            draft_only: None,
+            envs: vec![],
+            is_template: None,
+            kind: None,
+            parent_hash: None,
+            lock: vec![],
+            summary: "".to_string(),
+            tag: None,
+            schema: std::collections::HashMap::new(),
+        },
+    ).await.unwrap();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        completed.next().await; // deployed script
+
+        let script =
+            query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string())
+                .fetch_one(&db2)
+                .await
+                .unwrap();
+
+        let job = RunJob::from(JobPayload::ScriptHash {
+            path: "f/system/test_import".to_string(),
+            hash: ScriptHash(script.hash),
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+            dedicated_worker: None,
+            language,
+        }).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    let db2 = db.clone();
+    in_test_worker(&db, async move {
+        let job = RunJob::from(JobPayload::Code(RawCode {
+            content: script_content,
+            path: Some("f/system/test_import".to_string()),
+            language,
+            lock: None,
+            concurrent_limit: None,
+            concurrency_time_window_s: None,
+            cache_ttl: None,
+        })).push(&db2).await;
+
+
+        completed.next().await; // completed job
+
+
+        let result = completed_job(job, &db2).await.json_result().unwrap();
+
+
+        assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
+
+    }, port).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_relative_imports_bun(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_bun"))]
+async fn test_nested_imports_bun(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_relative_imports_deno(db: Pool<Postgres>) {
+    let content = r#"
+import { main as test1 } from "/f/system/same_folder_script.ts";
+import { main as test2 } from "./same_folder_script.ts";
+import { main as test3 } from "/f/system_relative/different_folder_script.ts";
+import { main as test4 } from "../system_relative/different_folder_script.ts";
+
+export async function main() {
+  return [test1(), test2(), test3(), test4()];
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_deno"))]
+async fn test_nested_imports_deno(db: Pool<Postgres>) {
+
+    let content = r#"
+import { main as test } from "/f/system_relative/nested_script.ts";
+
+export async function main() {
+  return test();
+}
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_relative_imports_python(db: Pool<Postgres>) {
+    let content = r#"
+from f.system.same_folder_script import main as test1
+from .same_folder_script import main as test2
+from f.system_relative.different_folder_script import main as test3
+from ..system_relative.different_folder_script import main as test4
+    
+def main():
+    return [test1(), test2(), test3(), test4()]
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+
+}
+
+
+#[sqlx::test(fixtures("base", "relative_python"))]
+async fn test_nested_imports_python(db: Pool<Postgres>) {
+
+    let content = r#"
+
+from f.system_relative.nested_script import main as test
+
+def main():
+    return test()
+"#.to_string();
+
+    run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
+    run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
+}

backend/windmill-api/Cargo.toml

@@ -53,7 +53,7 @@ cron.workspace = true
 mime_guess.workspace = true
 rust-embed.workspace = true
 tracing-subscriber.workspace = true
-retainer.workspace = true
+quick_cache.workspace = true
 rand.workspace = true
 time.workspace = true
 magic-crypt.workspace = true
@@ -74,3 +74,4 @@ bytes.workspace = true
 mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
+rsa.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.173.0
+  version: 1.177.0
   title: Windmill API
 
   contact:
@@ -575,6 +575,81 @@ paths:
             application/json:
               schema: {}
 
+  /settings/test_smtp:
+    post:
+      summary: test smtp
+      operationId: testSmtp
+      tags:
+        - setting
+      requestBody:
+        description: test smtp payload
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                to:
+                  type: string
+                smtp:
+                  type: object
+                  properties:
+                    host:
+                      type: string
+                    username:
+                      type: string
+                    password:
+                      type: string
+                    port:
+                      type: integer
+                    from:
+                      type: string
+                    tls_implicit:
+                      type: boolean
+                  required:
+                    - host
+                    - username
+                    - password
+                    - port
+                    - from
+                    - tls_implicit
+              required:
+                - to
+                - smtp
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
+  /settings/test_license_key:
+    post:
+      summary: test license key
+      operationId: testLicenseKey
+      tags:
+        - setting
+      requestBody:
+        description: test license key
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                license_key:
+                  type: string
+              required:
+                - license_key
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -3519,6 +3594,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
+  /w/{workspace}/apps_u/public_resource/{path}:
+    get:
+      summary: get public resource
+      operationId: get public resource
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/apps/secret_of/{path}:
     get:
       summary: get public secret of app
@@ -4052,6 +4143,7 @@ paths:
         - $ref: "#/components/parameters/StartedBefore"
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/CreatedOrStartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
@@ -5252,15 +5344,15 @@ paths:
                 items:
                   $ref: "#/components/schemas/WorkerPing"
 
-  /workers/list_worker_groups:
+  /configs/list_worker_groups:
     get:
-      summary: list workers
+      summary: list worker groups
       operationId: listWorkerGroups
       tags:
-        - worker
+        - config
       responses:
         "200":
-          description: a list of workers
+          description: a list of worker group configs
           content:
             application/json:
               schema:
@@ -5275,12 +5367,27 @@ paths:
                     - name
                     - config
 
-  /workers/worker_group/{name}:
-    post:
-      summary: Update Worker Group
-      operationId: updateWorkerGroup
+  /configs/get/{name}:
+    get:
+      summary: get config
+      operationId: get config
       tags:
-        - worker
+        - config
+      parameters:
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: a config
+          content:
+            application/json:
+              schema: {}
+
+  /configs/update/{name}:
+    post:
+      summary: Update config
+      operationId: updateConfig
+      tags:
+        - config
       parameters:
         - $ref: "#/components/parameters/Name"
       requestBody:
@@ -5297,15 +5404,15 @@ paths:
               schema:
                 type: string
     delete:
-      summary: Delete Worker Group
-      operationId: deleteWorkerGroup
+      summary: Delete Config
+      operationId: deleteConfig
       tags:
-        - worker
+        - config
       parameters:
         - $ref: "#/components/parameters/Name"
       responses:
         "200":
-          description: Delete a worker group
+          description: Delete config
           content:
             text/plain:
               schema:

backend/windmill-api/src/apps.rs

@@ -57,6 +57,7 @@ pub fn unauthed_service() -> Router {
     Router::new()
         .route("/execute_component/*path", post(execute_component))
         .route("/public_app/:secret", get(get_public_app_by_secret))
+        .route("/public_resource/*path", get(get_public_resource))
 }
 
 pub fn global_service() -> Router {
@@ -341,6 +342,27 @@ async fn get_public_app_by_secret(
     Ok(Json(app))
 }
 
+async fn get_public_resource(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    if !path.starts_with("f/app_themes/") {
+        return Err(Error::BadRequest(
+            "Only app themes are public resources".to_string(),
+        ));
+    }
+    let res = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&db)
+    .await?
+    .flatten();
+    Ok(Json(res))
+}
+
 async fn get_secret_id(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,

backend/windmill-api/src/capture.rs

@@ -9,7 +9,7 @@
 use axum::{
     extract::{Extension, Path, Query},
     routing::{get, post, put},
-    Json, Router,
+    Router,
 };
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
@@ -21,7 +21,7 @@ use windmill_common::{
 
 use crate::{
     db::{ApiAuthed, DB},
-    jobs::add_include_headers,
+    jobs::{add_include_headers, add_raw_string, JsonOrForm},
 };
 
 const KEEP_LAST: i64 = 8;
@@ -95,11 +95,12 @@ pub async fn update_payload(
     Path((w_id, path)): Path<(String, StripPath)>,
     Query(run_query): Query<IncludeHeaderQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> Result<StatusCode> {
     let mut tx = db.begin().await?;
-
     let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
+
     sqlx::query!(
         "
        UPDATE capture

backend/windmill-api/src/configs.rs

@@ -0,0 +1,114 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use axum::{
+    extract::{Extension, Path},
+    routing::{get, post},
+    Json, Router,
+};
+
+use serde::{Deserialize, Serialize};
+use sqlx::FromRow;
+use windmill_common::{
+    db::UserDB,
+    error::{self},
+    DB,
+};
+
+use crate::{db::ApiAuthed, utils::require_super_admin};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/list_worker_groups", get(list_worker_groups))
+        .route("/update/:name", post(update_config).delete(delete_config))
+        .route("/get/:name", get(get_config))
+}
+
+#[derive(Serialize, Deserialize, FromRow)]
+struct Config {
+    name: String,
+    config: serde_json::Value,
+}
+
+async fn list_worker_groups(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Vec<Config>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
+        .fetch_all(&db)
+        .await?;
+    Ok(Json(rows))
+}
+
+async fn get_config(
+    authed: ApiAuthed,
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Option<serde_json::Value>> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
+        .fetch_optional(&db)
+        .await?
+        .map(|c| c.config);
+
+    Ok(Json(config))
+}
+
+async fn update_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(config): Json<serde_json::Value>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    #[cfg(not(feature = "enterprise"))]
+    if name.starts_with("worker__") {
+        return Err(error::Error::BadRequest(
+            "Worker groups configurable from UI available only in the enterprise version"
+                .to_string(),
+        ));
+    }
+
+    sqlx::query!(
+        "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
+        &name,
+        config
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Updated config {name}"))
+}
+
+async fn delete_config(
+    Path(name): Path<String>,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    authed: ApiAuthed,
+) -> error::Result<String> {
+    let tx = user_db.begin(&authed).await?;
+
+    require_super_admin(&db, &authed.email).await?;
+    tx.commit().await?;
+
+    let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name)
+        .fetch_all(&db)
+        .await?;
+
+    if deleted.len() == 0 {
+        return Err(error::Error::NotFound(format!(
+            "Config {name} not found",
+            name = name
+        )));
+    }
+    Ok(format!("Deleted config {name}"))
+}

backend/windmill-api/src/ee.rs

@@ -0,0 +1,41 @@
+use anyhow::anyhow;
+use base64::Engine;
+use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
+use sha2::Sha256;
+
+pub async fn validate_license_key(license_key: String) -> anyhow::Result<String> {
+    let mut splitted_lk = license_key.split(".");
+    if splitted_lk.clone().count() != 3 {
+        return Err(anyhow!(
+            "license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)"
+        ));
+    }
+
+    let id = splitted_lk.next().unwrap();
+    let expiry = splitted_lk.next().unwrap();
+    let signature_b64 = splitted_lk.next().unwrap();
+
+    const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
+    let pub_key = rsa::RsaPublicKey::from_public_key_der(
+        &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
+    )?;
+    let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
+    rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
+        .verify(
+            &format!("{id}{expiry}").as_bytes(),
+            &rsa::pss::Signature::from(signature),
+        )
+        .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
+
+    let expiry_nb = expiry.parse::<u64>()?;
+    if expiry_nb < chrono::Utc::now().timestamp() as u64 {
+        tracing::error!(
+            "License key expired: {} < {}",
+            expiry_nb,
+            chrono::Utc::now().timestamp() as u64
+        );
+        return Err(anyhow!("License key expired".to_string()));
+    };
+
+    Ok(id.to_string())
+}

backend/windmill-api/src/jobs.rs

@@ -13,7 +13,6 @@ use crate::{
     users::{check_scopes, require_owner_of_path, OptAuthed},
     utils::require_super_admin,
     variables::get_workspace_key,
-    BASE_URL,
 };
 use anyhow::Context;
 use axum::{
@@ -24,6 +23,7 @@ use axum::{
 };
 use base64::Engine;
 use bytes::Bytes;
+use chrono::Utc;
 use hmac::Mac;
 use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
@@ -33,7 +33,8 @@ use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
 use tower_http::cors::{Any, CorsLayer};
 use urlencoding::encode;
 use windmill_audit::{audit_log, ActionKind};
-use windmill_common::worker::CUSTOM_TAGS_PER_WORKSPACE;
+use windmill_common::worker::{CUSTOM_TAGS_PER_WORKSPACE, SERVER_CONFIG};
+use windmill_common::BASE_URL;
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error},
@@ -41,7 +42,7 @@ use windmill_common::{
     flows::FlowValue,
     jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
     oauth2::HmacSha256,
-    scripts::{ScriptHash, ScriptLang},
+    scripts::{Script, ScriptHash, ScriptLang},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
@@ -105,7 +106,7 @@ pub fn workspaced_service() -> Router {
                 .layer(cors.clone()),
         )
         .route("/run/preview", post(run_preview_job))
-        .route("/add_noop_jobs/:n", post(add_noop_jobs))
+        .route("/add_batch_jobs/:n", post(add_batch_jobs))
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
         .route("/queue/list", get(list_queue_jobs))
@@ -779,7 +780,14 @@ async fn list_jobs(
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
 
-    let sqlc = list_completed_jobs_query(
+    if lq.success.is_some() && lq.running.is_some_and(|x| x) {
+        return Err(error::Error::BadRequest(
+            "cannot specify both success and running".to_string(),
+        ));
+
+    }
+    let sqlc = if lq.running.is_none() {  
+        Some(list_completed_jobs_query(
         &w_id,
         per_page + offset,
         0,
@@ -816,7 +824,10 @@ async fn list_jobs(
             "null as concurrent_limit",
             "null as concurrency_time_window_s",
         ],
-    );
+        ))
+    } else {
+        None
+    };
 
     let sql = if lq.success.is_none() {
         let sqlq = list_queue_jobs_query(
@@ -832,7 +843,7 @@ async fn list_jobs(
                 created_after: lq.created_after,
                 created_or_started_before: lq.created_or_started_before,
                 created_or_started_after: lq.created_or_started_after,
-                running: None,
+                running: lq.running,
                 parent_job: lq.parent_job,
                 order_desc: Some(true),
                 job_kinds: lq.job_kinds,
@@ -875,6 +886,7 @@ async fn list_jobs(
             ],
         );
 
+        if let Some(sqlc) = sqlc {
         format!(
             "{} UNION ALL {} LIMIT {} OFFSET {};",
             &sqlq.subquery()?,
@@ -882,8 +894,11 @@ async fn list_jobs(
             per_page,
             offset
         )
+        } else {
+            sqlq.query()?
+        }
     } else {
-        sqlc.query()?
+        sqlc.unwrap().query()?
     };
     let mut tx = user_db.begin(&authed).await?;
     let jobs: Vec<UnifiedJob> = sqlx::query_as(&sql).fetch_all(&mut *tx).await?;
@@ -1295,7 +1310,8 @@ pub async fn get_resume_urls(
         .map(|x| format!("?approver={}", encode(x)))
         .unwrap_or_else(String::new);
 
-    let base_url = BASE_URL.as_str();
+    let base_url_str = BASE_URL.read().await.clone();
+    let base_url = base_url_str.as_str();
     let res = ResumeUrls {
         approvalPage: format!(
             "{base_url}/approve/{w_id}/{job_id}/{resume_id}/{signature}{approver}"
@@ -1480,8 +1496,8 @@ struct PreviewFlow {
 }
 
 pub struct JsonOrForm(
-    Option<serde_json::Map<String, serde_json::Value>>,
-    Option<String>,
+    pub Option<serde_json::Map<String, serde_json::Value>>,
+    pub Option<String>,
 );
 
 #[axum::async_trait]
@@ -1594,7 +1610,7 @@ fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
     serde_json::from_slice(vec.as_slice()).context("invalid json")
 }
 
-fn add_raw_string(
+pub fn add_raw_string(
     raw_string: Option<String>,
     mut args: serde_json::Map<String, serde_json::Value>,
 ) -> serde_json::Map<String, serde_json::Value> {
@@ -1634,6 +1650,19 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) ->
     }
 }
 
+#[cfg(feature = "enterprise")]
+pub async fn check_license_key_valid() -> error::Result<()> {
+    use crate::LICENSE_KEY_VALID;
+
+    let valid = *LICENSE_KEY_VALID.read().await;
+    if !valid {
+        return Err(error::Error::BadRequest(format!(
+            "License key is not valid. Go to your superadmin settings to update your license key.",
+        )));
+    }
+    Ok(())
+}
+
 pub async fn run_flow_by_path(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -1644,6 +1673,8 @@ pub async fn run_flow_by_path(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
     let flow_path = flow_path.to_path();
     check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
 
@@ -1697,7 +1728,11 @@ pub async fn run_job_by_path(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     let script_path = script_path.to_path();
+
     check_scopes(&authed, || format!("run:script/{script_path}"))?;
 
     let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
@@ -1774,11 +1809,11 @@ impl Drop for Guard {
 async fn run_wait_result<T>(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
-    timeout: i32,
     uuid: Uuid,
     Path((w_id, _)): Path<(String, T)>,
 ) -> error::JsonResult<serde_json::Value> {
     let mut result;
+    let timeout = SERVER_CONFIG.read().await.timeout_wait_result.clone();
     let timeout_ms = if timeout <= 0 {
         2000
     } else {
@@ -1862,10 +1897,6 @@ lazy_static::lazy_static! {
     pub static ref QUEUE_LIMIT_WAIT_RESULT: Option<i64> = std::env::var("QUEUE_LIMIT_WAIT_RESULT")
         .ok()
         .and_then(|x| x.parse().ok());
-    pub static ref TIMEOUT_WAIT_RESULT: i32 = std::env::var("TIMEOUT_WAIT_RESULT")
-        .ok()
-        .and_then(|x| x.parse().ok())
-        .unwrap_or(20);
     pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
         .ok()
         .and_then(|x| x.parse().ok())
@@ -1889,6 +1920,9 @@ pub async fn run_wait_result_job_by_path_get(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
 ) -> error::JsonResult<serde_json::Value> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     if method == http::Method::HEAD {
         return Ok(Json(serde_json::json!("")));
     }
@@ -1936,14 +1970,7 @@ pub async fn run_wait_result_job_by_path_get(
     .await?;
     tx.commit().await?;
 
-    run_wait_result(
-        authed,
-        Extension(user_db),
-        *TIMEOUT_WAIT_RESULT,
-        uuid,
-        Path((w_id, script_path)),
-    )
-    .await
+    run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await
 }
 
 pub async fn run_wait_result_flow_by_path_get(
@@ -1956,6 +1983,9 @@ pub async fn run_wait_result_flow_by_path_get(
     headers: HeaderMap,
     Query(run_query): Query<RunJobQuery>,
 ) -> error::JsonResult<serde_json::Value> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     if method == http::Method::HEAD {
         return Ok(Json(serde_json::json!("")));
     }
@@ -1995,6 +2025,9 @@ pub async fn run_wait_result_script_by_path(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     run_wait_result_script_by_path_internal(
         db,
         run_query,
@@ -2107,14 +2140,7 @@ async fn run_wait_result_script_by_path_internal(
     .await?;
     tx.commit().await?;
 
-    run_wait_result(
-        authed,
-        Extension(user_db),
-        *TIMEOUT_WAIT_RESULT,
-        uuid,
-        Path((w_id, script_path)),
-    )
-    .await
+    run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await
 }
 
 pub async fn run_wait_result_script_by_hash(
@@ -2127,6 +2153,9 @@ pub async fn run_wait_result_script_by_hash(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     check_queue_too_long(&db, run_query.queue_limit).await?;
 
     let hash = script_hash.0;
@@ -2179,14 +2208,7 @@ pub async fn run_wait_result_script_by_hash(
     .await?;
     tx.commit().await?;
 
-    run_wait_result(
-        authed,
-        Extension(user_db),
-        *TIMEOUT_WAIT_RESULT,
-        uuid,
-        Path((w_id, script_hash)),
-    )
-    .await
+    run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await
 }
 
 pub async fn openai_sync_flow_by_path(
@@ -2224,6 +2246,9 @@ pub async fn run_wait_result_flow_by_path(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     run_wait_result_flow_by_path_internal(
         db, run_query, flow_path, authed, rsmq, user_db, headers, args, raw_string, w_id,
     )
@@ -2286,14 +2311,7 @@ async fn run_wait_result_flow_by_path_internal(
     .await?;
     tx.commit().await?;
 
-    run_wait_result(
-        authed,
-        Extension(user_db),
-        *TIMEOUT_WAIT_RESULT,
-        uuid,
-        Path((w_id, flow_path)),
-    )
-    .await
+    run_wait_result(authed, Extension(user_db), uuid, Path((w_id, flow_path))).await
 }
 
 async fn run_preview_job(
@@ -2306,6 +2324,9 @@ async fn run_preview_job(
     headers: HeaderMap,
     Json(preview): Json<Preview>,
 ) -> error::Result<(StatusCode, String)> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     check_scopes(&authed, || format!("runscript"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
@@ -2357,53 +2378,137 @@ async fn run_preview_job(
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
+#[derive(Deserialize)]
+struct BatchInfo {
+    kind: String,
+    flow_value: Option<FlowValue>,
+    path: Option<String>,
+}
+
 #[tracing::instrument(level = "trace", skip_all)]
-async fn add_noop_jobs(
+async fn add_batch_jobs(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Path((w_id, n)): Path<(String, i32)>,
-) -> error::JsonResult<Vec<String>> {
+    Json(batch_info): Json<BatchInfo>,
+) -> error::JsonResult<Vec<Uuid>> {
     require_super_admin(&db, &authed.email).await?;
-    let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
 
-    let mut uuids: Vec<String> = Vec::new();
-    for _ in 0..n {
-        let (uuid, ntx) = push(
-            &db,
-            tx,
-            &w_id,
-            JobPayload::Noop,
-            serde_json::Map::new(),
-            &authed.username,
-            &authed.email,
-            username_to_permissioned_as(&authed.username),
-            None,
-            None,
-            None,
-            None,
-            None,
-            false,
-            false,
-            None,
-            true,
-            None,
-            None,
-            None,
-        )
-        .await?;
-        tx = PushIsolationLevel::Transaction(ntx);
-        uuids.push(uuid.to_string());
-    }
-    match tx {
-        PushIsolationLevel::Transaction(tx) => {
-            tx.commit().await?;
+    let (hash, path, job_kind, language, dedicated_worker) = match batch_info.kind.as_str() {
+        "script" => {
+            let script = sqlx::query_as::<_, Script>(
+                "select * from script where path = $1 and workspace_id = $2",
+            )
+            .bind(&batch_info.path)
+            .bind(&w_id)
+            .fetch_optional(&db)
+            .await?
+            .ok_or_else(|| {
+                error::Error::BadRequest(format!("Script not found: {:?}", batch_info.path))
+            })?;
+            (
+                Some(script.hash),
+                batch_info.path,
+                JobKind::Script,
+                Some(script.language),
+                script.dedicated_worker,
+            )
         }
-        _ => (),
-    }
+        "flow" => {
+            let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
+
+            let mut uuids: Vec<Uuid> = Vec::new();
+            if batch_info.flow_value.is_none() {
+                return Err(error::Error::BadRequest(
+                    "Flow value is required for batch flow".to_string(),
+                ));
+            }
+            for _ in 0..n {
+                let (uuid, ntx) = push(
+                    &db,
+                    tx,
+                    &w_id,
+                    JobPayload::RawFlow {
+                        value: batch_info.flow_value.clone().unwrap(),
+                        path: None,
+                    },
+                    serde_json::Map::new(),
+                    &authed.username,
+                    &authed.email,
+                    username_to_permissioned_as(&authed.username),
+                    None,
+                    None,
+                    None,
+                    None,
+                    None,
+                    false,
+                    false,
+                    None,
+                    true,
+                    None,
+                    None,
+                    None,
+                )
+                .await?;
+                tx = PushIsolationLevel::Transaction(ntx);
+                uuids.push(uuid);
+            }
+            match tx {
+                PushIsolationLevel::Transaction(tx) => {
+                    tx.commit().await?;
+                }
+                _ => (),
+            }
+            return Ok(Json(uuids));
+        }
+        "noop" => (None, None, JobKind::Noop, None, None),
+        _ => {
+            return Err(error::Error::BadRequest(format!(
+                "Invalid batch kind: {}",
+                batch_info.kind
+            )))
+        }
+    };
+
+    let language = language.unwrap_or(ScriptLang::Deno);
+
+    let tag = if let Some(dedicated_worker) = dedicated_worker {
+        if dedicated_worker && path.is_some() {
+            format!("{}:{}", w_id, path.clone().unwrap())
+        } else {
+            format!("{}", language.as_str())
+        }
+    } else {
+        format!("{}", language.as_str())
+    };
+
+    let uuids = sqlx::query_scalar!(
+        r#"WITH uuid_table as (
+            select gen_random_uuid() as uuid from generate_series(1, $11)
+        )
+        INSERT INTO queue 
+            (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)
+            (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) 
+        RETURNING id"#,
+            hash.map(|h| h.0),
+            path,
+            job_kind.clone() as JobKind,
+            language as ScriptLang,
+            tag,
+            authed.username,
+            username_to_permissioned_as(&authed.username),
+            authed.email,
+            Utc::now(),
+            w_id,
+            n
+        )
+        .fetch_all(&db)
+        .await?;
 
     Ok(Json(uuids))
 }
+
 async fn run_preview_flow_job(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -2464,6 +2569,9 @@ pub async fn run_job_by_hash(
     headers: HeaderMap,
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
     let hash = script_hash.0;
     let (
         path,
@@ -2667,6 +2775,7 @@ pub struct ListCompletedQuery {
     pub created_or_started_before: Option<chrono::DateTime<chrono::Utc>>,
     pub created_or_started_after: Option<chrono::DateTime<chrono::Utc>>,
     pub success: Option<bool>,
+    pub running: Option<bool>,
     pub parent_job: Option<String>,
     pub order_desc: Option<bool>,
     pub job_kinds: Option<String>,

backend/windmill-api/src/lib.rs

@@ -12,7 +12,7 @@ use crate::saml::{SamlSsoLogin, ServiceProviderExt};
 use crate::scim::has_scim_token;
 use crate::tracing_init::MyOnFailure;
 use crate::{
-    oauth2::{build_oauth_clients, SlackVerifier},
+    oauth2::SlackVerifier,
     tracing_init::{MyMakeSpan, MyOnResponse},
     users::OptAuthed,
     webhook_util::WebhookShared,
@@ -24,9 +24,10 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
 use hyper::{http, Method};
-use mail_send::SmtpClientBuilder;
 use reqwest::Client;
+use std::collections::HashMap;
 use std::{net::SocketAddr, sync::Arc};
+use tokio::sync::RwLock;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
 use tower_http::{
@@ -36,14 +37,17 @@ use tower_http::{
 use windmill_common::db::UserDB;
 use windmill_common::utils::rd_string;
 use windmill_common::worker::ALL_TAGS;
+use windmill_common::BASE_URL;
 
 use windmill_common::error::AppError;
 
 mod apps;
 mod audit;
 mod capture;
+mod configs;
 mod db;
 mod drafts;
+pub mod ee;
 mod favorite;
 mod flows;
 mod folders;
@@ -51,7 +55,7 @@ mod granular_acls;
 mod groups;
 mod inputs;
 pub mod jobs;
-mod oauth2;
+pub mod oauth2;
 mod openai;
 mod raw_apps;
 mod resources;
@@ -72,16 +76,11 @@ mod workspaces;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-pub use users::delete_expired_items;
+pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
 
-pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
 lazy_static::lazy_static! {
-    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
-    pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
-    .ok()
-    .and_then(|x| x.parse::<usize>().ok())
-    .unwrap_or(DEFAULT_BODY_LIMIT);
+    pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
 
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
@@ -90,59 +89,22 @@ lazy_static::lazy_static! {
         .ok()
         .map(|x| SlackVerifier::new(x).unwrap());
 
-        static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
+    pub static ref IS_SECURE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
 
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
         .user_agent("windmill/beta")
         .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
         .build().unwrap();
 
-    pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
-        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
-        .unwrap();
+    pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
+        logins: HashMap::new(),
+        connects: HashMap::new(),
+        slack: None
+    }));
 
-    pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
-        let smtp = parse_smtp();
-        if let Some(smtp) = smtp {
-            match smtp {
-                Ok(smtp) => Some(smtp),
-                Err(e) => {
-                    tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e);
-                    None
-                }
-            }
-        } else {
-            tracing::warn!("SMTP is not configured, emails will not be sent");
-            None
-        }
-    };
-
-    pub static ref SMTP_FROM: String = std::env::var("SMTP_FROM").unwrap_or_else(|_| "noreply@getwindmill.com".to_string());
-
-    pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
-}
-
-pub fn parse_smtp() -> Option<windmill_common::error::Result<SmtpClientBuilder<String>>> {
-    let username = std::env::var("SMTP_USERNAME").ok();
-    let port = std::env::var("SMTP_PORT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(587);
-    let password = std::env::var("SMTP_PASSWORD").ok();
-    let host = std::env::var("SMTP_HOST").ok();
-    let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT")
-        .ok()
-        .and_then(|p| p.parse().ok())
-        .unwrap_or(false);
-
-    if username.is_some() && password.is_some() && host.is_some() {
-        let smtp = SmtpClientBuilder::new(host.unwrap(), port)
-            .implicit_tls(tls_implicit)
-            .credentials((username.unwrap(), password.unwrap()));
-        Some(Ok(smtp))
-    } else {
-        None
-    }
+    pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+    pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+    pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 }
 
 pub async fn run_server(
@@ -185,7 +147,9 @@ pub async fn run_server(
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
-        .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
+        .layer(DefaultBodyLimit::max(
+            REQUEST_SIZE_LIMIT.read().await.clone(),
+        ));
 
     let cors = CorsLayer::new()
         .allow_methods([Method::GET, Method::POST])
@@ -239,6 +203,7 @@ pub async fn run_server(
                 .nest("/settings", settings::global_service())
                 .nest("/jobs", jobs::global_root_service())
                 .nest("/workers", workers::global_service())
+                .nest("/configs", configs::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/groups", groups::global_service())
                 .nest("/flows", flows::global_service())
@@ -287,15 +252,16 @@ pub async fn run_server(
 
     let instance_name = rd_string(5);
 
-    tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening");
     let server = axum::Server::bind(&addr).serve(app.into_make_service());
 
     let port = server.local_addr().port();
     tracing::info!(
+        instance = %instance_name,
         "server started on port={} and addr={}",
         port,
         server.local_addr().ip()
     );
+
     port_tx
         .send(server.local_addr().port())
         .expect("Failed to send port");
@@ -305,8 +271,6 @@ pub async fn run_server(
         println!("Graceful shutdown of server");
     });
 
-    tokio::spawn(async move { auth_cache.monitor().await });
-
     server.await?;
     Ok(())
 }
@@ -354,13 +318,7 @@ async fn ee_license() -> &'static str {
 
 #[cfg(feature = "enterprise")]
 async fn ee_license() -> String {
-    LICENSE_KEY
-        .as_ref()
-        .unwrap()
-        .split(".")
-        .next()
-        .unwrap()
-        .to_string()
+    LICENSE_KEY_ID.read().await.clone()
 }
 
 async fn openapi() -> &'static str {

backend/windmill-api/src/oauth2.rs

@@ -81,7 +81,7 @@ pub fn workspaced_service() -> Router {
         .route("/connect_slack_callback", post(connect_slack_callback))
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ClientWithScopes {
     client: OClient,
     scopes: Vec<String>,
@@ -120,7 +120,10 @@ pub struct AllClients {
     pub slack: Option<OClient>,
 }
 
-pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
+pub fn build_oauth_clients(
+    base_url: &str,
+    oauths_from_config: Option<HashMap<String, OAuthClient>>,
+) -> anyhow::Result<AllClients> {
     let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
         "../../oauth_connect.json"
     ))?;
@@ -128,26 +131,36 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         "../../oauth_login.json"
     ))?;
 
-    let path = "./oauth.json";
-    let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
-        str::from_utf8(
-            &base64::engine::general_purpose::STANDARD
-                .decode(e)
-                .map_err(to_anyhow)?,
-        )?
-        .to_string()
-    } else if std::path::Path::new(path).exists() {
-        fs::read_to_string(path).map_err(to_anyhow)?
+    let oauths = if let Some(oauths) = oauths_from_config {
+        oauths
     } else {
-        tracing::warn!("oauth.json not found, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
+        let path = "./oauth.json";
+        let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
+            str::from_utf8(
+                &base64::engine::general_purpose::STANDARD
+                    .decode(e)
+                    .map_err(to_anyhow)?,
+            )?
+            .to_string()
+        } else if std::path::Path::new(path).exists() {
+            fs::read_to_string(path).map_err(to_anyhow)?
+        } else {
+            tracing::warn!("oauth.json not found, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
 
-    if content.is_empty() {
-        tracing::warn!("oauth.json is empty, no OAuth clients loaded");
-        return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
-    };
-    let oauths: HashMap<String, OAuthClient> =
+        if content.is_empty() {
+            tracing::warn!("oauth.json is empty, no OAuth clients loaded");
+            return Ok(AllClients {
+                logins: HashMap::new(),
+                connects: HashMap::new(),
+                slack: None,
+            });
+        };
         match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
             Ok(clients) => clients,
             Err(e) => {
@@ -156,7 +169,8 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             }
         }
         .into_iter()
-        .collect();
+        .collect()
+    };
 
     tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", "));
 
@@ -168,7 +182,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -177,17 +191,25 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 base_url,
                 None,
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: client_params.allowed_domains.clone(),
-                    userinfo_url: config.userinfo_url,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: client_params.allowed_domains.clone(),
+                            userinfo_url: config.userinfo_url,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
@@ -199,7 +221,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 .as_ref()
                 .map(|c| (x.0.clone(), (x.1, c.clone())))
         }))
-        .map(|(k, (client_params, config))| {
+        .filter_map(|(k, (client_params, config))| {
             let named_client = build_basic_client(
                 k.clone(),
                 config.clone(),
@@ -212,43 +234,61 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                     None
                 },
             );
-            (
-                named_client.0,
-                ClientWithScopes {
-                    client: named_client.1,
-                    scopes: config.scopes.unwrap_or(vec![]),
-                    extra_params: config.extra_params,
-                    extra_params_callback: config.extra_params_callback,
-                    allowed_domains: None,
-                    userinfo_url: None,
-                },
-            )
+            named_client
+                .map(|named_client| {
+                    (
+                        named_client.0,
+                        ClientWithScopes {
+                            client: named_client.1,
+                            scopes: config.scopes.unwrap_or(vec![]),
+                            extra_params: config.extra_params,
+                            extra_params_callback: config.extra_params_callback,
+                            allowed_domains: None,
+                            userinfo_url: None,
+                        },
+                    )
+                })
+                .map_err(|e| {
+                    tracing::error!("Error building oauth client {k}: {e}");
+                    e
+                })
+                .ok()
         })
         .collect();
 
-    let slack = oauths.get("slack").map(|v| {
-        build_basic_client(
-            "slack".to_string(),
-            OAuthConfig {
-                auth_url: "https://slack.com/oauth/authorize".to_string(),
-                token_url: "https://slack.com/api/oauth.access".to_string(),
-                userinfo_url: None,
-                scopes: None,
-                extra_params: None,
-                extra_params_callback: None,
-                req_body_auth: None,
-            },
-            v.clone(),
-            false,
-            base_url,
-            Some(format!("{base_url}/oauth/callback_slack")),
-        )
-        .1
-    });
-
-    Ok(AllClients { logins, connects, slack })
+    let slack = oauths
+        .get("slack")
+        .map(|v| {
+            build_basic_client(
+                "slack".to_string(),
+                OAuthConfig {
+                    auth_url: "https://slack.com/oauth/authorize".to_string(),
+                    token_url: "https://slack.com/api/oauth.access".to_string(),
+                    userinfo_url: None,
+                    scopes: None,
+                    extra_params: None,
+                    extra_params_callback: None,
+                    req_body_auth: None,
+                },
+                v.clone(),
+                false,
+                base_url,
+                Some(format!("{base_url}/oauth/callback_slack")),
+            )
+            .map(|x| x.1)
+            .map_err(|e| {
+                tracing::error!("Error building oauth slack client: {e}");
+                e
+            })
+            .ok()
+        })
+        .flatten();
+    let all_clients = AllClients { logins, connects, slack };
+    tracing::info!("Final oauth config: {all_clients:#?}");
+    Ok(all_clients)
 }
 
+use anyhow::anyhow;
 pub fn build_basic_client(
     name: String,
     config: OAuthConfig,
@@ -256,9 +296,11 @@ pub fn build_basic_client(
     login: bool,
     base_url: &str,
     override_callback: Option<String>,
-) -> (String, OClient) {
-    let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
-    let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL");
+) -> error::Result<(String, OClient)> {
+    let auth_url = Url::parse(&config.auth_url)
+        .map_err(|e| anyhow!("Invalid authorization endpoint URL: {e}"))?;
+    let token_url =
+        Url::parse(&config.token_url).map_err(|e| anyhow!("Invalid token endpoint URL: {e}"))?;
 
     let redirect_url = if login {
         format!("{base_url}/user/login_callback/{name}")
@@ -273,9 +315,12 @@ pub fn build_basic_client(
         client.set_auth_type(AuthType::RequestBody);
     }
     client.set_client_secret(client_params.secret.clone());
-    client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL"));
+    client.set_redirect_url(
+        Url::parse(&redirect_url).map_err(|e| anyhow!("Invalid redirect URL: {e}"))?,
+    );
+
     // Set up the config for the Github OAuth2 process.
-    (name.to_string(), client)
+    Ok((name.to_string(), client))
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -314,7 +359,7 @@ async fn connect(
     cookies: Cookies,
 ) -> error::Result<Redirect> {
     let mut query = query.clone();
-    let connects = &OAUTH_CLIENTS.connects;
+    let connects = &OAUTH_CLIENTS.read().await.connects;
     let scopes = query
         .get("scopes")
         .map(|x| x.split('+').map(|x| x.to_owned()).collect());
@@ -330,7 +375,7 @@ async fn connect(
         cookies,
         scopes,
         extra_params,
-        *IS_SECURE,
+        IS_SECURE.read().await.clone(),
     )
 }
 
@@ -406,6 +451,8 @@ struct Logins {
 async fn list_logins(Extension(sso): Extension<Arc<SamlSsoLogin>>) -> error::JsonResult<Logins> {
     Ok(Json(Logins {
         oauth: OAUTH_CLIENTS
+            .read()
+            .await
             .logins
             .keys()
             .map(|x| x.to_owned())
@@ -421,7 +468,7 @@ struct ScopesAndParams {
 }
 async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
     Ok(Json(
-        (&OAUTH_CLIENTS.connects)
+        (&OAUTH_CLIENTS.read().await.connects)
             .into_iter()
             .map(|(k, v)| {
                 (
@@ -438,6 +485,8 @@ async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>>
 
 async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     let mut client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -453,7 +502,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
     client.add_scope("commands");
     let url = client.authorize_url(&state);
 
-    set_cookie(&state, cookies, *IS_SECURE);
+    set_cookie(&state, cookies, IS_SECURE.read().await.clone());
     Ok(Redirect::to(url.as_str()))
 }
 
@@ -496,8 +545,15 @@ async fn disconnect_slack(
 }
 
 async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
-    let clients = &OAUTH_CLIENTS.logins;
-    oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
+    let clients = &OAUTH_CLIENTS.read().await.logins;
+    oauth_redirect(
+        clients,
+        client_name,
+        cookies,
+        None,
+        None,
+        IS_SECURE.read().await.clone(),
+    )
 }
 
 #[derive(Deserialize)]
@@ -532,6 +588,8 @@ pub async fn _refresh_token<'c>(
     .await?;
     let account = not_found_if_none(account, "Account", &id.to_string())?;
     let client = (&OAUTH_CLIENTS
+        .read()
+        .await
         .connects
         .get(&account.client)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
@@ -622,11 +680,10 @@ async fn connect_callback(
     Path(client_name): Path<String>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::JsonResult<TokenResponse> {
-    let client_w_scopes = OAUTH_CLIENTS
-        .connects
+    let connects = &OAUTH_CLIENTS.read().await.connects;
+    let client_w_scopes = connects
         .get(&client_name)
         .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
-
     let client = client_w_scopes.client.to_owned();
     let extra_params = client_w_scopes.extra_params_callback.clone();
     let token_response =
@@ -644,6 +701,8 @@ async fn connect_slack_callback(
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
     let client = OAUTH_CLIENTS
+        .read()
+        .await
         .slack
         .as_ref()
         .ok_or_else(|| {
@@ -858,7 +917,7 @@ async fn slack_command(
                 None,
             )
             .await?;
-            let url = BASE_URL.to_owned();
+            let url = BASE_URL.read().await.clone();
             tx.commit().await?;
             return Ok(format!(
                 "Job launched. See details at {url}/run/{uuid}?workspace={}",
@@ -880,10 +939,13 @@ async fn login_callback(
     Extension(webhook): Extension<WebhookShared>,
     Json(callback): Json<OAuthCallback>,
 ) -> error::Result<String> {
-    let client_w_config = &OAUTH_CLIENTS
-        .logins
-        .get(&client_name)
-        .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
+    let client_w_config = {
+        let clients = OAUTH_CLIENTS.read().await.logins.clone();
+        clients
+            .get(&client_name)
+            .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
+            .clone()
+    };
     let client = client_w_config.client.to_owned();
     let token_res =
         exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;

backend/windmill-api/src/saml.rs

@@ -71,7 +71,7 @@ pub async fn build_sp_extension() -> anyhow::Result<(ServiceProviderExt, SamlSso
                 ..ContactPerson::default()
             })
             .idp_metadata(idp_metadata)
-            .acs_url(format!("{}/api/saml/acs", *BASE_URL))
+            .acs_url(format!("{}/api/saml/acs", BASE_URL.read().await.clone()))
             .build()?;
 
         tracing::info!("SAML Configured, sso login link at: {:?}", url);

backend/windmill-api/src/settings.rs

@@ -6,8 +6,11 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use std::time::Duration;
+
 use crate::{
     db::{ApiAuthed, DB},
+    ee::validate_license_key,
     utils::require_super_admin,
 };
 
@@ -17,18 +20,74 @@ use axum::{
     Json, Router,
 };
 
+use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
+use serde::Deserialize;
+use tokio::time::timeout;
 use windmill_common::{
-    error::{self, JsonResult},
+    error::{self, to_anyhow, JsonResult},
     global_settings::ENV_SETTINGS,
+    server::Smtp,
 };
 
 pub fn global_service() -> Router {
     Router::new()
-        .route("/local", get(get_local_settings))
+        .route("/envs", get(get_local_settings))
         .route(
             "/global/:key",
             post(set_global_setting).get(get_global_setting),
         )
+        .route("/test_smtp", post(test_email))
+        .route("/test_license_key", post(test_license_key))
+}
+
+#[derive(Deserialize)]
+pub struct TestEmail {
+    pub to: String,
+    pub smtp: Smtp,
+}
+
+pub async fn test_email(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(test_email): Json<TestEmail>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    let smtp = test_email.smtp;
+    let to = test_email.to;
+    let client = SmtpClientBuilder::new(smtp.host, smtp.port)
+        .implicit_tls(smtp.tls_implicit)
+        .credentials((smtp.username, smtp.password));
+    let message = MessageBuilder::new()
+        .from(("Windmill", smtp.from.as_str()))
+        .to(to.clone())
+        .subject("Test email from Windmill")
+        .text_body("Test email content");
+    let dur = Duration::from_secs(3);
+    timeout(dur, client.connect())
+        .await
+        .map_err(to_anyhow)?
+        .map_err(to_anyhow)?
+        .send(message)
+        .await
+        .map_err(to_anyhow)?;
+    tracing::info!("Sent test email to {to}");
+    Ok("Sent test email".to_string())
+}
+
+#[derive(Deserialize)]
+pub struct TestKey {
+    pub license_key: String,
+}
+
+
+pub async fn test_license_key(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(TestKey { license_key }): Json<TestKey>,
+) -> error::Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    validate_license_key(license_key).await?;
+    Ok("Sent test email".to_string())
 }
 
 pub async fn get_local_settings(

backend/windmill-api/src/users.rs

@@ -8,7 +8,7 @@
 
 #![allow(non_snake_case)]
 
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
 use crate::db::ApiAuthed;
 
@@ -18,7 +18,7 @@ use crate::{
     utils::require_super_admin,
     webhook_util::{InstanceEvent, WebhookShared},
     workspaces::invite_user_to_all_auto_invite_worspaces,
-    BASE_URL, COOKIE_DOMAIN, IS_SECURE, SMTP_CLIENT, SMTP_FROM,
+    BASE_URL, COOKIE_DOMAIN, IS_SECURE,
 };
 use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
 use axum::{
@@ -32,16 +32,18 @@ use axum::{
 use hyper::{header::LOCATION, StatusCode};
 use lazy_static::lazy_static;
 use mail_send::mail_builder::MessageBuilder;
+use mail_send::SmtpClientBuilder;
+use quick_cache::sync::Cache;
 use rand::rngs::OsRng;
 use regex::Regex;
-use retainer::Cache;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use time::OffsetDateTime;
 use tower_cookies::{Cookie, Cookies};
 use tracing::{Instrument, Span};
 use windmill_audit::{audit_log, ActionKind};
-use windmill_common::worker::CLOUD_HOSTED;
+use windmill_common::users::truncate_token;
+use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG};
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
@@ -49,7 +51,6 @@ use windmill_common::{
     utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
 };
 
-const TTL_TOKEN_CACHE_S: u64 = 60; // 60s
 pub const TTL_TOKEN_DB_H: u32 = 72;
 
 const COOKIE_NAME: &str = "token";
@@ -102,19 +103,24 @@ pub fn make_unauthed_service() -> Router {
         .route("/logout", get(logout))
 }
 
+#[derive(Clone)]
+pub struct ExpiringAuthCache {
+    pub authed: ApiAuthed,
+    pub expiry: Option<chrono::DateTime<chrono::Utc>>,
+}
 pub struct AuthCache {
-    cache: Cache<(String, String), ApiAuthed>,
+    cache: Cache<(String, String), ExpiringAuthCache>,
     db: DB,
     superadmin_secret: Option<String>,
 }
 
 impl AuthCache {
     pub fn new(db: DB, superadmin_secret: Option<String>) -> Self {
-        AuthCache { cache: Cache::new(), db, superadmin_secret }
+        AuthCache { cache: Cache::new(300), db, superadmin_secret }
     }
 
     pub async fn invalidate(&self, w_id: &str, token: String) {
-        self.cache.remove(&(w_id.to_string(), token)).await;
+        self.cache.remove(&(w_id.to_string(), token));
     }
 
     pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
@@ -122,13 +128,17 @@ impl AuthCache {
             w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
             token.to_string(),
         );
-        let s = self.cache.get(&key).await.map(|c| c.to_owned());
+        let s = self.cache.get(&key).map(|c| c.to_owned());
         match s {
-            a @ Some(_) => a,
-            None => {
-                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>)>(
+            Some(ExpiringAuthCache { authed, expiry })
+                if expiry.is_none() || expiry.unwrap() > chrono::Utc::now() =>
+            {
+                Some(authed)
+            }
+            _ => {
+                let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>, Option<chrono::DateTime<chrono::Utc>>)>(
                     "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
-                     OR expiration IS NULL) RETURNING owner, email, super_admin, scopes",
+                     OR expiration IS NULL) RETURNING owner, email, super_admin, scopes, expiration",
                 )
                 .bind(token)
                 .fetch_optional(&self.db)
@@ -139,7 +149,7 @@ impl AuthCache {
                 if let Some(user) = user_o {
                     let authed_o = {
                         match user {
-                            (Some(owner), Some(email), super_admin, _) if w_id.is_some() => {
+                            (Some(owner), Some(email), super_admin, _, _) if w_id.is_some() => {
                                 if let Some((prefix, name)) = owner.split_once('/') {
                                     if prefix == "u" {
                                         let (is_admin, is_operator) = if super_admin {
@@ -218,7 +228,7 @@ impl AuthCache {
                                     })
                                 }
                             }
-                            (_, Some(email), super_admin, scopes) => {
+                            (_, Some(email), super_admin, scopes, _) => {
                                 if w_id.is_some() {
                                     let row_o = sqlx::query_as::<_, (String, bool, bool)>(
                                         "SELECT username, is_admin, operator FROM usr where email = $1 AND \
@@ -288,9 +298,10 @@ impl AuthCache {
                         }
                     };
                     if let Some(authed) = authed_o.as_ref() {
-                        self.cache
-                            .insert(key, authed.clone(), Duration::from_secs(TTL_TOKEN_CACHE_S))
-                            .await;
+                        self.cache.insert(
+                            key,
+                            ExpiringAuthCache { authed: authed.clone(), expiry: user.4 },
+                        );
                     }
                     authed_o
                 } else if self
@@ -314,10 +325,6 @@ impl AuthCache {
             }
         }
     }
-
-    pub async fn monitor(&self) {
-        self.cache.monitor(20, 0.25, Duration::from_secs(10)).await;
-    }
 }
 
 async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
@@ -1219,13 +1226,7 @@ async fn decline_invite(
 }
 
 lazy_static! {
-    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
-
-    pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
-    .ok()
-    .and_then(|x| x.parse::<u32>().ok())
-    .unwrap_or(60 * 60 * 24 * 60); // 60 days
-
+    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap();
 }
 
 async fn accept_invite(
@@ -1236,7 +1237,7 @@ async fn accept_invite(
 ) -> Result<(StatusCode, String)> {
     if !VALID_USERNAME.is_match(&nu.username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1319,7 +1320,7 @@ async fn add_user_to_workspace<'c>(
 
     if !VALID_USERNAME.is_match(username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1570,7 +1571,9 @@ async fn create_user(
 Log in and change your password: {}/user/login?email={}&password={}&rd=%2F%23user-settings
 
 You can then join or create a workspace. Happy building!",
-            *BASE_URL, &nu.email, &nu.password
+            BASE_URL.read().await.clone(),
+            &nu.email,
+            &nu.password
         ),
         &nu.email,
     );
@@ -1590,13 +1593,17 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
 }
 
 pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &str) -> Result<()> {
-    if let Some(ref smtp) = *SMTP_CLIENT {
+    if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() {
+        let client = SmtpClientBuilder::new(smtp.host, smtp.port)
+            .implicit_tls(smtp.tls_implicit)
+            .credentials((smtp.username, smtp.password));
         let message = MessageBuilder::new()
-            .from(("Windmill", SMTP_FROM.as_str()))
+            .from(("Windmill", smtp.from.as_str()))
             .to(to)
             .subject(subject)
             .text_body(content);
-        smtp.connect()
+        client
+            .connect()
             .await
             .map_err(to_anyhow)?
             .send(message)
@@ -1942,7 +1949,7 @@ pub async fn create_session_token<'c>(
     .execute(&mut **tx)
     .await?;
     let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
-    cookie.set_secure(*IS_SECURE);
+    cookie.set_secure(IS_SECURE.read().await.clone());
     cookie.set_same_site(Some(cookie::SameSite::Lax));
     cookie.set_http_only(true);
     cookie.set_path(COOKIE_PATH);
@@ -2358,65 +2365,3 @@ pub async fn login_externally(
     tx.commit().await?;
     Ok(())
 }
-
-pub async fn delete_expired_items(db: &DB) -> () {
-    let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-        "DELETE FROM token WHERE expiration <= now()
-        RETURNING concat(substring(token for 10), '*****')",
-    )
-    .fetch_all(db)
-    .await;
-
-    match tokens_deleted_r {
-        Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
-        Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
-    }
-
-    let pip_resolution_r = sqlx::query_scalar!(
-        "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
-    )
-    .fetch_all(db)
-    .await;
-
-    match pip_resolution_r {
-        Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
-        Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
-    }
-
-    let deleted_cache = sqlx::query_scalar!(
-            "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
-        )
-        .fetch_all(db)
-        .await;
-
-    match deleted_cache {
-        Ok(res) => tracing::debug!("deleted {} cache resource: {:?}", res.len(), res),
-        Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
-    }
-
-    if *JOB_RETENTION_SECS > 0 {
-        let deleted_jobs = sqlx::query_scalar!(
-                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
-                *JOB_RETENTION_SECS as i64
-            )
-            .fetch_all(db)
-            .await;
-
-        match deleted_jobs {
-            Ok(deleted_jobs) => {
-                tracing::info!(
-                    "deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
-                    deleted_jobs.len(),
-                    deleted_jobs
-                )
-            }
-            Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
-        }
-    }
-}
-
-pub fn truncate_token(token: &str) -> String {
-    let mut s = token[..10].to_owned();
-    s.push_str("*****");
-    s
-}

backend/windmill-api/src/variables.rs

@@ -68,6 +68,7 @@ async fn list_contextual_variables(
             Some("u/user/triggering_flow_path".to_string()),
             Some("c".to_string()),
         )
+        .await
         .to_vec(),
     ))
 }

backend/windmill-api/src/webhook_util.rs

@@ -1,7 +1,8 @@
 use std::time::Duration;
 
+use quick_cache::sync::Cache;
 use serde::Serialize;
-use tokio::{select, sync::mpsc, time::interval};
+use tokio::{select, sync::mpsc};
 use windmill_common::METRICS_ENABLED;
 
 use crate::db::DB;
@@ -78,8 +79,7 @@ impl WebhookShared {
                 .timeout(Duration::from_secs(5))
                 .build()
                 .unwrap();
-            let cache = retainer::Cache::new();
-            let mut cache_purge_interval = interval(Duration::from_secs(30));
+            let cache = Cache::new(100);
 
             loop {
                 select! {
@@ -87,7 +87,7 @@ impl WebhookShared {
                     _ = shutdown_rx.recv() => break,
                     r = rx.recv() => match r {
                         Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
-                            let url_guard = match cache.get(&workspace_id).await {
+                            let webhook_opt = match cache.get(&workspace_id) {
                                 Some(guard) => {
                                     guard
                                 },
@@ -104,16 +104,14 @@ impl WebhookShared {
                                             tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}");
                                             continue;
                                         };
-                                    cache.insert(workspace_id.clone(), webook_opt, Duration::from_secs(30)).await;
-                                    cache.get(&workspace_id).await.unwrap()
+                                    cache.insert(workspace_id, webook_opt.clone());
+                                    webook_opt
                                 }
                             };
-                            let webook_opt = url_guard.value();
-                            if let Some(url) = webook_opt {
+                            if let Some(url) = webhook_opt {
                                 let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
                                 let _ = client.post(url).json(&message).send().await;
                                 timer.map(|x| x.stop_and_record());
-                                drop(url_guard);
                             }
                         },
                         Some(WebhookPayload::InstanceEvent(event)) => {
@@ -125,10 +123,6 @@ impl WebhookShared {
                         },
                         None => break,
                     },
-                    _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
-                        tracing::trace!("Purging Webhook Cache");
-                        cache.purge(10, 0.50).await;
-                    },
                 }
             }
         });

backend/windmill-api/src/workers.rs

@@ -7,7 +7,7 @@
  */
 
 use axum::{
-    extract::{Extension, Path, Query},
+    extract::{Extension, Query},
     routing::get,
     Json, Router,
 };
@@ -16,35 +16,17 @@ use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use windmill_common::{
     db::UserDB,
-    error::{self, JsonResult},
+    error::JsonResult,
     utils::{paginate, Pagination},
     worker::ALL_TAGS,
-    DB,
 };
 
-#[cfg(feature = "benchmark")]
-use std::sync::atomic::Ordering;
-#[cfg(feature = "benchmark")]
-use windmill_queue::IDLE_WORKERS;
-
-use crate::{db::ApiAuthed, utils::require_super_admin};
+use crate::db::ApiAuthed;
 
 pub fn global_service() -> Router {
-    use axum::routing::post;
-
-    let router = Router::new()
+    Router::new()
         .route("/list", get(list_worker_pings))
         .route("/custom_tags", get(get_custom_tags))
-        .route("/list_worker_groups", get(get_worker_groups))
-        .route(
-            "/worker_group/:name",
-            post(update_worker_group).delete(delete_worker_group),
-        );
-    #[cfg(feature = "benchmark")]
-    return router.route("/toggle", get(toggle));
-
-    #[cfg(not(feature = "benchmark"))]
-    return router;
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -85,82 +67,6 @@ async fn list_worker_pings(
     Ok(Json(rows))
 }
 
-#[cfg(feature = "benchmark")]
-async fn toggle(Query(query): Query<EnableWorkerQuery>) -> JsonResult<bool> {
-    IDLE_WORKERS.store(query.disable, Ordering::Relaxed);
-    Ok(Json(IDLE_WORKERS.load(Ordering::Relaxed)))
-}
-
 async fn get_custom_tags() -> Json<Vec<String>> {
     Json(ALL_TAGS.read().await.clone().into())
 }
-
-#[derive(Serialize, Deserialize, FromRow)]
-struct WorkerGroup {
-    name: String,
-    config: serde_json::Value,
-}
-
-async fn get_worker_groups(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-) -> error::JsonResult<Vec<WorkerGroup>> {
-    require_super_admin(&db, &authed.email).await?;
-
-    let rows = sqlx::query_as!(WorkerGroup, "SELECT * FROM worker_group_config")
-        .fetch_all(&db)
-        .await?;
-    Ok(Json(rows))
-}
-
-#[cfg(feature = "enterprise")]
-async fn update_worker_group(
-    Path(name): Path<String>,
-    Extension(db): Extension<DB>,
-    authed: ApiAuthed,
-    Json(config): Json<serde_json::Value>,
-) -> error::Result<String> {
-    require_super_admin(&db, &authed.email).await?;
-
-    sqlx::query!(
-        "INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
-        &name,
-        config
-    )
-    .execute(&db)
-    .await?;
-
-    Ok(format!("Updated worker group {name}"))
-}
-
-#[cfg(not(feature = "enterprise"))]
-async fn update_worker_group() -> String {
-    "Worker groups available only in enterprise version".to_string()
-}
-
-async fn delete_worker_group(
-    Path(name): Path<String>,
-    Extension(db): Extension<DB>,
-    Extension(user_db): Extension<UserDB>,
-    authed: ApiAuthed,
-) -> error::Result<String> {
-    let tx = user_db.begin(&authed).await?;
-
-    require_super_admin(&db, &authed.email).await?;
-    tx.commit().await?;
-
-    let deleted = sqlx::query!(
-        "DELETE FROM worker_group_config WHERE name = $1 RETURNING name",
-        name,
-    )
-    .fetch_all(&db)
-    .await?;
-
-    if deleted.len() == 0 {
-        return Err(error::Error::NotFound(format!(
-            "Worker group {name} not found",
-            name = name
-        )));
-    }
-    Ok(format!("Deleted worker group {name}"))
-}

backend/windmill-api/src/workspaces.rs

@@ -284,8 +284,9 @@ async fn stripe_checkout(
         require_admin(authed.is_admin, &authed.username)?;
 
         let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-        let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL);
-        let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL);
+        let base_url = BASE_URL.read().await.clone();
+        let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url);
+        let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url);
         let checkout_session = {
             let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd);
             params.mode = Some(stripe::CheckoutSessionMode::Subscription);
@@ -332,7 +333,7 @@ async fn stripe_portal(
     .await?
     .ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?;
     let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
-    let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL);
+    let success_rd = format!("{}/workspace_settings?tab=premium", BASE_URL.read().await.clone());
     let portal_session = {
         let customer_id = CustomerId::from_str(&customer_id).unwrap();
         let mut params = stripe::CreateBillingPortalSession::new(customer_id);
@@ -845,7 +846,7 @@ async fn check_name_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str)
 
 lazy_static::lazy_static! {
 
-    pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(false));
+    pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(true));
 
 }
 
@@ -950,6 +951,21 @@ async fn create_workspace(
     .execute(&mut *tx)
     .await?;
 
+    sqlx::query!(
+        "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
+        nw.id,
+    )
+    .execute(&mut *tx)
+    .await?;
+
+
+    sqlx::query!(
+        "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
+        nw.id,
+    )
+    .execute(&mut *tx)
+    .await?;
+
     audit_log(
         &mut *tx,
         &authed.username,
@@ -1094,6 +1110,10 @@ async fn delete_workspace(
         .execute(&mut *tx)
         .await?;
 
+    sqlx::query!("DELETE FROM resource_type WHERE workspace_id = $1", &w_id)
+        .execute(&mut *tx)
+        .await?;
+
     sqlx::query!(
         "DELETE FROM workspace_invite WHERE workspace_id = $1",
         &w_id
@@ -1202,7 +1222,7 @@ async fn invite_user(
             "You have been granted access to Windmill's workspace {w_id}
 
 If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
-            *BASE_URL
+            BASE_URL.read().await.clone()
         ),
         &nu.email,
     );
@@ -1231,7 +1251,7 @@ async fn add_user(
     let mut tx = db.begin().await?;
     if !VALID_USERNAME.is_match(&nu.username) {
         return Err(windmill_common::error::Error::BadRequest(format!(
-            "Usermame can only contain alphanumeric characters and underscores"
+            "Usermame can only contain alphanumeric characters and underscores and must start with a letter"
         )));
     }
 
@@ -1266,7 +1286,7 @@ async fn add_user(
             "You have been granted access to Windmill's workspace {w_id} by {email}
 
 If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
-            *BASE_URL
+            BASE_URL.read().await.clone()
         ),
         &nu.email,
     );

backend/windmill-common/Cargo.toml

@@ -45,4 +45,4 @@ tracing-subscriber = { workspace = true, optional = true }
 lazy_static.workspace = true
 tracing-flame = { version = "^0", optional = true }
 itertools.workspace = true
-regex.workspace = true
+regex.workspace = true

backend/windmill-common/src/global_settings.rs

@@ -1,5 +1,10 @@
 pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync";
 pub const CUSTOM_TAGS_SETTING: &str = "custom_tags";
+pub const BASE_URL_SETTING: &str = "base_url";
+pub const OAUTH_SETTING: &str = "oauths";
+pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
+pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
+pub const LICENSE_KEY_SETTING: &str = "license_key";
 
 pub const ENV_SETTINGS: [&str; 54] = [
     "DISABLE_NSJAIL",

backend/windmill-common/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::SocketAddr;
+use std::{net::SocketAddr, sync::Arc};
 
 use error::Error;
 use scripts::ScriptLang;
@@ -24,6 +24,7 @@ pub mod more_serde;
 pub mod oauth2;
 pub mod schedule;
 pub mod scripts;
+pub mod server;
 pub mod users;
 pub mod utils;
 pub mod variables;
@@ -47,7 +48,7 @@ lazy_static::lazy_static! {
     .flatten()
     .flatten();
     pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
-    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
+    pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
     pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 }
 
@@ -78,6 +79,7 @@ pub async fn shutdown_signal(
     Ok(())
 }
 
+use tokio::sync::RwLock;
 #[cfg(feature = "prometheus")]
 use tokio::task::JoinHandle;
 

backend/windmill-common/src/scripts.rs

@@ -161,6 +161,7 @@ pub struct Script {
     pub concurrent_limit: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_time_window_s: Option<i32>,
+    pub dedicated_worker: Option<bool>,
 }
 
 #[derive(Serialize)]

backend/windmill-common/src/server.rs

@@ -0,0 +1,112 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{error, DB};
+
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+pub struct Smtp {
+    pub host: String,
+    pub username: String,
+    pub password: String,
+    pub port: u16,
+    pub from: String,
+    pub tls_implicit: bool,
+}
+
+#[derive(Serialize, Deserialize, PartialEq)]
+pub struct ServerConfigOpt {
+    pub smtp_host: Option<String>,
+    pub smtp_username: Option<String>,
+    pub smtp_password: Option<String>,
+    pub smtp_port: Option<u16>,
+    pub smtp_from: Option<String>,
+    pub smtp_tls_implicit: Option<bool>,
+    pub timeout_wait_result: Option<u64>,
+}
+
+pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
+    let config: ServerConfigOpt =
+        sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",)
+            .fetch_optional(db)
+            .await?
+            .flatten()
+            .map(|x| serde_json::from_value(x).ok())
+            .flatten()
+            .unwrap_or_default();
+
+    let config_smtp = if let (Some(host), Some(username), Some(password)) =
+        (config.smtp_host, config.smtp_username, config.smtp_password)
+    {
+        Some(Smtp {
+            host,
+            username,
+            password,
+            tls_implicit: config.smtp_tls_implicit.unwrap_or(false),
+            port: config.smtp_port.unwrap_or(587),
+            from: config
+                .smtp_from
+                .unwrap_or_else(|| "noreply@getwindmill.com".to_string()),
+        })
+    } else {
+        None
+    };
+    let smtp = config_smtp.or(
+        if let (Some(host), Some(username), Some(password)) = (
+            std::env::var("SMTP_HOST").ok(),
+            std::env::var("SMTP_USERNAME").ok(),
+            std::env::var("SMTP_PASSWORD").ok(),
+        ) {
+            Some(Smtp {
+                host,
+                username,
+                password,
+                tls_implicit: std::env::var("SMTP_TLS_IMPLICIT")
+                    .ok()
+                    .and_then(|p| p.parse().ok())
+                    .unwrap_or(false),
+                port: std::env::var("SMTP_PORT")
+                    .ok()
+                    .and_then(|p| p.parse().ok())
+                    .unwrap_or(587),
+                from: std::env::var("SMTP_FROM")
+                    .unwrap_or_else(|_| "noreply@getwindmill.com".to_string()),
+            })
+        } else {
+            None
+        },
+    );
+    if smtp.is_none() {
+        tracing::warn!("SMTP not configured");
+    }
+
+    Ok(ServerConfig {
+        smtp,
+        timeout_wait_result: config
+            .timeout_wait_result
+            .ok_or(
+                std::env::var("TIMEOUT_WAIT_RESULT")
+                    .ok()
+                    .and_then(|x| x.parse::<u64>().ok()),
+            )
+            .unwrap_or(600),
+    })
+}
+
+impl Default for ServerConfigOpt {
+    fn default() -> Self {
+        Self {
+            smtp_from: None,
+            smtp_host: None,
+            smtp_password: None,
+            smtp_port: None,
+            smtp_tls_implicit: None,
+            smtp_username: None,
+            timeout_wait_result: Default::default(),
+        }
+    }
+}
+
+#[derive(PartialEq, Clone, Debug)]
+pub struct ServerConfig {
+    pub smtp: Option<Smtp>,
+    pub timeout_wait_result: u64,
+}

backend/windmill-common/src/users.rs

@@ -15,3 +15,9 @@ pub fn username_to_permissioned_as(user: &str) -> String {
         format!("u/{}", user)
     }
 }
+
+pub fn truncate_token(token: &str) -> String {
+    let mut s = token[..10].to_owned();
+    s.push_str("*****");
+    s
+}

backend/windmill-common/src/variables.rs

@@ -61,7 +61,7 @@ pub struct CreateVariable {
     pub is_oauth: Option<bool>,
 }
 
-pub fn get_reserved_variables(
+pub async fn get_reserved_variables(
     w_id: &str,
     token: &str,
     email: &str,
@@ -163,7 +163,7 @@ pub fn get_reserved_variables(
         },
         ContextualVariable {
             name: "WM_BASE_URL".to_string(),
-            value: BASE_URL.clone(),
+            value: BASE_URL.read().await.clone(),
             description: "base url of this instance".to_string(),
         },
         ContextualVariable {

backend/windmill-common/src/worker.rs

@@ -5,7 +5,7 @@ use regex::Regex;
 use serde::{Deserialize, Serialize};
 use tokio::sync::RwLock;
 
-use crate::{error, global_settings::CUSTOM_TAGS_SETTING, DB};
+use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
 
 lazy_static::lazy_static! {
     pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -35,6 +35,9 @@ lazy_static::lazy_static! {
         dedicated_worker: Default::default(),
     }));
 
+    pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
+
+
 
     pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
 
@@ -75,15 +78,12 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
 
     let custom_tags = process_custom_tags(tags);
 
-    {
-        let l = CUSTOM_TAGS_PER_WORKSPACE.read().await;
-        if l.clone() == custom_tags {
-            tracing::info!("Custom tags setting unchanged, skipping update");
-            return Ok(());
-        } else {
-            tracing::info!("Custom tags setting changed, updating");
-        }
-    }
+    tracing::info!(
+        "Loaded setting custom tags, common: {:?}, per-workspace: {:?}",
+        custom_tags.0,
+        custom_tags.1,
+    );
+
     {
         let mut l = CUSTOM_TAGS_PER_WORKSPACE.write().await;
         *l = custom_tags.clone()
@@ -97,10 +97,6 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
         .concat();
     }
     Ok(())
-    // pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec<String>, HashMap<String, Vec<String>>) =  process_custom_tags(std::env::var("CUSTOM_TAGS")
-    //     .ok());
-
-    // pub static ref ALL_TAGS: Vec<String> = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat();
 }
 
 fn process_custom_tags(tags: Vec<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
@@ -119,16 +115,23 @@ fn process_custom_tags(tags: Vec<String>) -> (Vec<String>, HashMap<String, Vec<S
 }
 
 pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
-    let wc = WORKER_CONFIG.read().await;
-    let tags = wc.worker_tags.as_slice();
+    let (tags, dw) = {
+        let wc = WORKER_CONFIG.read().await.clone();
+        (
+            wc.worker_tags,
+            wc.dedicated_worker
+                .as_ref()
+                .map(|x| format!("{}:{}", x.workspace_id, x.path)),
+        )
+    };
     sqlx::query!(
         "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
         worker_instance,
         worker_name,
         ip,
-        tags,
+        tags.as_slice(),
         *WORKER_GROUP,
-        wc.dedicated_worker.as_ref().map(|x| format!("{}:{}", x.workspace_id, x.path))
+        dw
     )
     .execute(db)
     .await
@@ -136,9 +139,9 @@ pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db:
 }
 
 pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
-    let config: WorkerConfigOpt = sqlx::query_scalar!(
-        "SELECT config FROM worker_group_config WHERE name = $1",
-        *WORKER_GROUP
+    let mut config: WorkerConfigOpt = sqlx::query_scalar!(
+        "SELECT config FROM config WHERE name = $1",
+        format!("worker__{}", *WORKER_GROUP)
     )
     .fetch_optional(db)
     .await?
@@ -146,6 +149,9 @@ pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
     .map(|x| serde_json::from_value(x).ok())
     .flatten()
     .unwrap_or_default();
+    if config.dedicated_worker.is_none() {
+        config.dedicated_worker = std::env::var("DEDICATED_WORKER").ok();
+    }
     let dedicated_worker = config.dedicated_worker.map(|x| {
         let splitted = x.split(':').to_owned().collect_vec();
         if splitted.len() != 2 {
@@ -193,7 +199,7 @@ impl Default for WorkerConfigOpt {
     }
 }
 
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone)]
 pub struct WorkerConfig {
     pub worker_tags: Vec<String>,
     pub dedicated_worker: Option<WorkspacedPath>,

backend/windmill-queue/src/jobs.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{collections::HashMap, sync::atomic::AtomicBool, vec};
+use std::{collections::HashMap, vec};
 
 use anyhow::Context;
 use async_recursion::async_recursion;
@@ -70,10 +70,6 @@ lazy_static::lazy_static! {
     )
     .unwrap();
 
-    // When compiled in 'benchmark' mode, this flags is exposed via the /workers/toggle endpoint
-    // and make it possible to disable to current active workers (such that they don't pull any)
-    // jobs from the queue
-    pub static ref IDLE_WORKERS: AtomicBool = AtomicBool::new(false);
 }
 
 #[cfg(feature = "enterprise")]
@@ -1081,7 +1077,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
                 , started_at = null
                 , scheduled_for = '{estimated_next_schedule_timestamp}'
                 , logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END
-                WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false)
+                WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())
                 RETURNING *"
             ))
             .fetch_all(&mut tx)
@@ -1164,9 +1160,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
          *   suspend_until is non-null
          *   and suspend = 0 when the resume messages are received
          *   or suspend_until <= now() if it has timed out */
-        let config = WORKER_CONFIG.read().await;
-        let tags = config.worker_tags.as_slice();
-
+        let config = WORKER_CONFIG.read().await.clone();
+        let tags = config.worker_tags.clone();
+        drop(config);
         let r = if suspend_first {
             sqlx::query_as::<_, QueuedJob>("UPDATE queue
             SET running = true
@@ -1188,14 +1184,12 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
         } else {
             None
         };
-        drop(config);
 
         if r.is_none() {
             // #[cfg(feature = "benchmark")]
             // let instant = Instant::now();
 
-            let config = WORKER_CONFIG.read().await;
-            let tags = config.worker_tags.as_slice();
+            let tags = WORKER_CONFIG.read().await.worker_tags.clone();
 
             let r = sqlx::query_as::<_, QueuedJob>(
                 "UPDATE queue
@@ -1750,11 +1744,11 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             tag = None;
         }
         let default = || {
-            if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
+            if job_kind == JobKind::Flow
+                || job_kind == JobKind::FlowPreview
+                || job_kind == JobKind::Identity
+            {
                 "flow".to_string()
-            } else if job_kind == JobKind::Identity {
-                // identity is a light script, nativets is too
-                "nativets".to_string()
             } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
                 "dependency".to_string()
             } else {

backend/windmill-worker/loader.bun.ts

@@ -24,15 +24,25 @@ const p = {
         loader: "tsx",
       };
     });
-    build.onResolve({ filter: /(?!\.\/main\.ts)\..*\.ts$/ }, (args) => {
-      const cdir = resolve("./");
+    const cdir = resolve("./");
+    const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos
+    const filter = new RegExp(
+      `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.ts).*\\.ts$`
+    );
+    build.onResolve({ filter }, (args) => {
       const file_path =
         args.importer == "./main.ts" || args.importer == resolve("./main.ts")
           ? current_path
           : args.importer.replace(cdir + "/", "");
 
-      const url = `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`;
-      const file = resolve("./" + current_path + "/../" + args.path + ".url");
+      const isRelative = !args.path.startsWith("/");
+
+      const url = isRelative
+        ? `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`
+        : `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${args.path}`;
+      const file = isRelative
+        ? resolve("./" + current_path + "/../" + args.path + ".url")
+        : resolve("./" + args.path + ".url");
       mkdirSync(dirname(file), { recursive: true });
       writeFileSync(file, url);
       return {

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -71,14 +71,6 @@ mount {
 }
 
 
-mount {
-    src: "{JOB_DIR}/node_modules"
-    dst: "/tmp/bun/node_modules"
-    is_bind: true
-    mandatory: false
-}
-
-
 mount {
     src: "{JOB_DIR}/wrapper.ts"
     dst: "/tmp/bun/wrapper.ts"
@@ -139,7 +131,6 @@ mount {
     src: "{CACHE_DIR}"
     dst: "/tmp/windmill/cache/bun"
     is_bind: true
-    rw: true
     mandatory: false
 }
 

backend/windmill-worker/src/bun_executor.rs

@@ -18,14 +18,17 @@ use crate::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
         write_file, write_file_binary,
     },
-    AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
+    AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV,
     NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
 
 #[cfg(feature = "enterprise")]
 use crate::MAX_BUFFERED_DEDICATED_JOBS;
 
-use tokio::{fs::File, process::Command};
+use tokio::{
+    fs::{remove_dir_all, File},
+    process::Command,
+};
 
 #[cfg(feature = "enterprise")]
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
@@ -216,17 +219,19 @@ pub async fn handle_bun_job(
                     })?,
             )
             .await?;
+
+            install_lockfile(
+                logs,
+                &job.id,
+                &job.workspace_id,
+                db,
+                job_dir,
+                worker_name,
+                common_bun_proc_envs.clone(),
+            )
+            .await?;
+            remove_dir_all(format!("{}/node_modules", job_dir)).await?;
         }
-        install_lockfile(
-            logs,
-            &job.id,
-            &job.workspace_id,
-            db,
-            job_dir,
-            worker_name,
-            common_bun_proc_envs.clone(),
-        )
-        .await?;
     } else if !*DISABLE_NSJAIL {
         logs.push_str("\n\n--- BUN INSTALL ---\n");
         set_logs(&logs, &job.id, &db).await;
@@ -243,6 +248,7 @@ pub async fn handle_bun_job(
             false,
         )
         .await?;
+        remove_dir_all(format!("{}/node_modules", job_dir)).await?;
     }
 
     logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
@@ -421,6 +427,7 @@ plugin(p)
 pub fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
     let mut deno_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("HOME"), HOME_ENV.clone()),
         (String::from("TZ"), TZ_ENV.clone()),
         (String::from("DISABLE_COLORS"), "0".to_string()),
         (String::from("DO_NOT_TRACK"), "1".to_string()),
@@ -458,6 +465,10 @@ pub async fn start_worker(
     mut jobs_rx: Receiver<QueuedJob>,
     mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
 ) -> Result<()> {
+    use std::task::Poll;
+
+    use futures::{future, Future};
+
     let mut logs = "".to_string();
     let _ = write_file(job_dir, "main.ts", inner_content).await?;
     let common_bun_proc_envs: HashMap<String, String> =
@@ -475,6 +486,7 @@ pub async fn start_worker(
         None,
         None,
     )
+    .await
     .to_vec();
     let context_envs = build_envs_map(context);
     if let Some(reqs) = requirements_o {
@@ -597,7 +609,8 @@ for await (const chunk of Bun.stdin.stream()) {{
         None,
         None,
         None,
-    );
+    )
+    .await;
 
     let _ = write_file(
         &job_dir,
@@ -676,6 +689,21 @@ plugin(p)
     // let mut i = 0;
     // let mut j = 0;
     let mut alive = true;
+
+    fn conditional_polling<T>(
+        fut: impl Future<Output = T>,
+        predicate: bool,
+    ) -> impl Future<Output = T> {
+        let mut fut = Box::pin(fut);
+        future::poll_fn(move |cx| {
+            if predicate {
+                fut.as_mut().poll(cx)
+            } else {
+                Poll::Pending
+            }
+        })
+    }
+
     loop {
         tokio::select! {
             biased;
@@ -703,8 +731,8 @@ plugin(p)
                     tracing::info!("dedicated worker process exited");
                     break;
                 }
-            }
-            job = jobs_rx.recv(), if alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS => {
+            },
+            job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => {
                 // i += 1;
                 if let Some(job) = job {
                     tracing::debug!("received job");

backend/windmill-worker/src/common.rs

@@ -192,6 +192,7 @@ pub async fn get_reserved_variables(
         job.schedule_path.clone(),
         job.flow_step_id.clone(),
     )
+    .await
     .to_vec();
 
     Ok(build_envs_map(variables))

backend/windmill-worker/src/deno_executor.rs

@@ -8,8 +8,8 @@ use crate::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
         write_file,
     },
-    AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, NPM_CONFIG_REGISTRY,
-    PATH_ENV, TZ_ENV,
+    AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
+    NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
 };
 use tokio::{fs::File, io::AsyncReadExt, process::Command};
 use windmill_common::{error::Result, BASE_URL};
@@ -41,8 +41,12 @@ lazy_static::lazy_static! {
 
 
 }
-fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<String, String> {
-    let hostname_base = BASE_URL.split("://").last().unwrap_or("localhost");
+async fn get_common_deno_proc_envs(
+    token: &str,
+    base_internal_url: &str,
+) -> HashMap<String, String> {
+    let hostname = BASE_URL.read().await.clone();
+    let hostname_base = hostname.split("://").last().unwrap_or("localhost");
     let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost");
     let deno_auth_tokens_base = DENO_AUTH_TOKENS.as_str();
     let deno_auth_tokens =
@@ -50,6 +54,7 @@ fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<St
 
     let mut deno_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("HOME"), HOME_ENV.clone()),
         (String::from("TZ"), TZ_ENV.clone()),
         (String::from("DENO_AUTH_TOKENS"), deno_auth_tokens),
         (
@@ -258,7 +263,7 @@ run().catch(async (e) => {{
         write_import_map_f
     )?;
 
-    let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url);
+    let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
 
     //do not cache local dependencies
     let reload = format!("--reload={base_internal_url}");

backend/windmill-worker/src/worker.rs

@@ -10,12 +10,16 @@ use anyhow::Result;
 use const_format::concatcp;
 use itertools::Itertools;
 use once_cell::sync::OnceCell;
+use prometheus::core::{AtomicU64, GenericCounter};
 #[cfg(feature = "benchmark")]
 use serde::Serialize;
 use sqlx::{Pool, Postgres};
 use std::{
     collections::HashMap,
-    sync::{atomic::Ordering, Arc},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
     time::Duration,
 };
 use windmill_api_client::Client;
@@ -62,9 +66,6 @@ use crate::bun_executor::start_worker;
 
 use windmill_queue::{add_completed_job, add_completed_job_error};
 
-#[cfg(feature = "benchmark")]
-use windmill_queue::IDLE_WORKERS;
-
 use crate::{
     bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE},
     bun_executor::{gen_lockfile, handle_bun_job},
@@ -340,6 +341,52 @@ macro_rules! add_time {
     };
 }
 
+async fn handle_receive_completed_job<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static,
+>(
+    jc: JobCompleted,
+    worker_execution_failed: HashMap<Option<ScriptLang>, GenericCounter<AtomicU64>>,
+    base_internal_url: String,
+    db: Pool<Postgres>,
+    worker_dir: String,
+    same_worker_tx: Sender<Uuid>,
+    rsmq: Option<R>,
+) {
+    let metrics = build_language_metrics(&worker_execution_failed.clone(), &jc.job.language);
+    let token = jc.token.clone();
+    let workspace = jc.job.workspace_id.clone();
+    let client = AuthedClient {
+        base_internal_url: base_internal_url.to_string(),
+        workspace,
+        token,
+        client: OnceCell::new(),
+    };
+    if let Err(err) = process_completed_job(
+        &jc,
+        &client,
+        &db,
+        &worker_dir,
+        metrics.clone(),
+        same_worker_tx.clone(),
+        rsmq.clone(),
+    )
+    .await
+    {
+        handle_job_error(
+            &db,
+            &client,
+            &jc.job,
+            err,
+            metrics,
+            false,
+            same_worker_tx.clone(),
+            &worker_dir,
+            rsmq.clone(),
+        )
+        .await;
+    }
+}
+
 pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
     db: &Pool<Postgres>,
     worker_instance: &str,
@@ -515,8 +562,6 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     #[cfg(feature = "enterprise")]
     let mut copy_cache_from_bucket_handle: Option<tokio::task::JoinHandle<()>> = None;
 
-    tracing::info!(worker = %worker_name, "starting worker");
-
     #[cfg(feature = "enterprise")]
     let mut last_sync = Instant::now()
         + Duration::from_secs(rand::thread_rng().gen_range(0..*GLOBAL_CACHE_INTERVAL));
@@ -552,43 +597,168 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     let rsmq2 = rsmq.clone();
     let worker_dir2 = worker_dir.clone();
     let worker_execution_failed2 = worker_execution_failed.clone();
+    let thread_count = Arc::new(AtomicUsize::new(0));
+
+    let is_dedicated_worker = WORKER_CONFIG.read().await.dedicated_worker.is_some();
+
+    #[cfg(feature = "benchmark")]
+    let jobs = 25000;
+
+    #[cfg(feature = "benchmark")]
+    {
+        if is_dedicated_worker {
+            // you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47
+            let hash = sqlx::query_scalar!(
+                "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
+                "f/benchmarks/dedicated",
+                "admins"
+            )
+            .fetch_one(db)
+            .await
+            .unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
+            sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+                    hash,
+                    "f/benchmarks/dedicated",
+                    JobKind::Script as JobKind,
+                    ScriptLang::Bun as ScriptLang,
+                    "admins:f/benchmarks/dedicated",
+                    "admin",
+                    "u/admin",
+                    "admin@windmill.dev",
+                    chrono::Utc::now(),
+                    "admins",
+                    jobs
+                )
+                .execute(db)
+                .await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
+        } else {
+            sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
+            None::<i64>,
+            None::<String>,
+            JobKind::Noop as JobKind,
+            ScriptLang::Deno as ScriptLang,
+            "deno",
+            "admin",
+            "u/admin",
+            "admin@windmill.dev",
+            chrono::Utc::now(),
+            "admins",
+            jobs
+        )
+        .execute(db)
+        .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs"));
+        }
+    }
+
+    #[cfg(feature = "benchmark")]
+    let completed_jobs = Arc::new(AtomicUsize::new(0));
+    #[cfg(feature = "benchmark")]
+    let start = Instant::now();
+    #[cfg(feature = "benchmark")]
+    let main_duration = Arc::new(AtomicUsize::new(0));
+    #[cfg(feature = "benchmark")]
+    let send_duration = Arc::new(AtomicUsize::new(0));
+    #[cfg(feature = "benchmark")]
+    let process_duration = Arc::new(AtomicUsize::new(0));
+
+    #[cfg(feature = "benchmark")]
+    let main_duration2 = main_duration.clone();
+    #[cfg(feature = "benchmark")]
+    let send_duration2 = send_duration.clone();
+
     let send_result = tokio::spawn(async move {
         while let Some(jc) = job_completed_rx.recv().await {
-            let metrics = build_language_metrics(&worker_execution_failed2, &jc.job.language);
-            let token = jc.token.clone();
-            let workspace = jc.job.workspace_id.clone();
-            let client = AuthedClient {
-                base_internal_url: base_internal_url2.to_string(),
-                workspace,
-                token,
-                client: OnceCell::new(),
-            };
-            if let Err(err) = process_completed_job(
-                &jc,
-                &client,
-                &db2,
-                &worker_dir2,
-                metrics.clone(),
-                same_worker_tx2.clone(),
-                rsmq2.clone(),
-            )
-            .await
-            {
-                handle_job_error(
-                    &db2,
-                    &client,
-                    &jc.job,
-                    err,
-                    metrics,
-                    false,
-                    same_worker_tx2.clone(),
-                    &worker_dir2,
-                    rsmq2.clone(),
+            let base_internal_url2 = base_internal_url2.clone();
+            let worker_execution_failed2 = worker_execution_failed2.clone();
+            let worker_dir2 = worker_dir2.clone();
+            let db2 = db2.clone();
+            let same_worker_tx2 = same_worker_tx2.clone();
+            let rsmq2 = rsmq2.clone();
+
+            if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker {
+                thread_count.fetch_add(1, Ordering::SeqCst);
+                let thread_count = thread_count.clone();
+
+                #[cfg(feature = "benchmark")]
+                let send_duration = send_duration2.clone();
+                #[cfg(feature = "benchmark")]
+                let process_duration = process_duration.clone();
+                #[cfg(feature = "benchmark")]
+                let completed_jobs = completed_jobs.clone();
+                #[cfg(feature = "benchmark")]
+                let main_duration = main_duration2.clone();
+
+                tokio::spawn(async move {
+                    #[cfg(feature = "benchmark")]
+                    let process_start = Instant::now();
+
+                    handle_receive_completed_job(
+                        jc,
+                        worker_execution_failed2,
+                        base_internal_url2,
+                        db2,
+                        worker_dir2,
+                        same_worker_tx2,
+                        rsmq2,
+                    )
+                    .await;
+                    #[cfg(feature = "benchmark")]
+                    {
+                        let n = completed_jobs.fetch_add(1, Ordering::SeqCst);
+                        if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize {
+                            let duration_s = start.elapsed().as_secs_f64();
+                            let jobs_per_sec = n as f64 / duration_s;
+                            tracing::info!(
+                                "completed {} jobs in {}s, {} jobs/s",
+                                n + 1,
+                                duration_s,
+                                jobs_per_sec
+                            );
+
+                            tracing::info!(
+                                "main loop without send {}s",
+                                main_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                            );
+
+                            tracing::info!(
+                                "send job completed / send dedicated job duration {}s",
+                                send_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                            );
+
+                            tracing::info!(
+                                "job completed process duration {}s",
+                                process_duration.load(Ordering::SeqCst) as f64 / 1000.0
+                            );
+                        }
+
+                        process_duration.fetch_add(
+                            process_start.elapsed().as_millis() as usize,
+                            Ordering::SeqCst,
+                        );
+                    }
+
+                    thread_count.fetch_sub(1, Ordering::SeqCst);
+                });
+            } else {
+                handle_receive_completed_job(
+                    jc,
+                    worker_execution_failed2,
+                    base_internal_url2,
+                    db2,
+                    worker_dir2,
+                    same_worker_tx2,
+                    rsmq2,
                 )
                 .await;
             }
         }
 
+        tracing::info!("stopped processing new completed jobs");
+        while thread_count.load(Ordering::SeqCst) > 0 {
+            tokio::time::sleep(Duration::from_millis(100)).await;
+        }
+        tracing::info!("finished processing all completed jobs");
+
         // if let Err(e) =
         //     add_completed_job(&db2, &job, success, false, result, logs, rsmq2.clone()).await
         // {
@@ -608,7 +778,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
     let vacuum_shift = rand::thread_rng().gen_range(0..VACUUM_PERIOD);
 
     IS_READY.store(true, Ordering::Relaxed);
-    tracing::info!(worker = %worker_name, "listening for jobs, config: {:#?}", WORKER_CONFIG.read().await);
+    tracing::info!(worker = %worker_name, "listening for jobs, config: {:?}", WORKER_CONFIG.read().await);
 
     let (dedicated_worker_tx, dedicated_worker_handle) = if let Some(_wp) =
         WORKER_CONFIG.read().await.dedicated_worker.clone()
@@ -722,6 +892,9 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
         (None, None) as (Option<Sender<QueuedJob>>, Option<JoinHandle<()>>)
     };
 
+    #[cfg(feature = "benchmark")]
+    tracing::info!("pre loop time {}s", start.elapsed().as_secs_f64());
+
     loop {
         #[cfg(feature = "benchmark")]
         let loop_start = Instant::now();
@@ -742,13 +915,12 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
         let copy_tx = _copy_to_bucket_tx.clone();
 
         if last_ping.elapsed().as_secs() > NUM_SECS_PING {
-            let wc = WORKER_CONFIG.read().await;
-            let tags = wc.worker_tags.as_slice();
+            let tags = WORKER_CONFIG.read().await.worker_tags.clone();
 
             sqlx::query!(
                 "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
                 jobs_executed,
-                tags,
+                tags.as_slice(),
                 &worker_name
             )
             .execute(db)
@@ -819,78 +991,67 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
         let next_job = {
             // println!("2: {:?}",  instant.elapsed());
-            let _wait_signal = false;
-
             #[cfg(feature = "benchmark")]
-            let _wait_signal = IDLE_WORKERS.load(Ordering::Relaxed);
+            if !started {
+                started = true
+            }
 
-            if _wait_signal {
-                // tracing::warn!("Worker is marked as idle. Not pulling any job for now");
-                tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await;
-                Ok(None)
-            } else {
-                #[cfg(feature = "benchmark")]
-                if !started {
-                    started = true
-                }
-
-                tokio::select! {
-                    biased;
-                    _ = killpill_rx.recv() => {
-                        #[cfg(feature = "enterprise")]
-                        if let Some(copy_cache_from_bucket_handle) = copy_cache_from_bucket_handle.as_ref() {
-                            if !copy_cache_from_bucket_handle.is_finished() {
-                                copy_cache_from_bucket_handle.abort();
-                            }
+            tokio::select! {
+                biased;
+                _ = killpill_rx.recv() => {
+                    #[cfg(feature = "enterprise")]
+                    if let Some(copy_cache_from_bucket_handle) = copy_cache_from_bucket_handle.as_ref() {
+                        if !copy_cache_from_bucket_handle.is_finished() {
+                            copy_cache_from_bucket_handle.abort();
                         }
-                        #[cfg(feature = "enterprise")]
-                        for handle in &handles {
-                            if !handle.is_finished() {
-                                handle.abort();
-                            }
+                    }
+                    #[cfg(feature = "enterprise")]
+                    for handle in &handles {
+                        if !handle.is_finished() {
+                            handle.abort();
                         }
-                        println!("received killpill for worker {}", i_worker);
-                        break
-                    },
-                    _ = copy_to_bucket_rx.recv() => {
-                        tracing::debug!("can_pull lock start");
-                        let _lock = CAN_PULL.write().await;
-                        // if num_workers > 1 {
-                        //     create_barrier_for_all_workers(num_workers, sync_barrier.clone()).await;
-                        // }
-                        //Arc::new(tokio::sync::Barrier::new(num_workers as usize + 1));
-                        #[cfg(feature = "enterprise")]
-                        if let Err(e) = copy_tmp_cache_to_cache().await {
-                            tracing::error!(worker = %worker_name, "failed to sync tmp cache to cache: {}", e);
-                        }
-                        tracing::debug!("can_pull lock end");
-                        Ok(None)
-                    },
-                    Some(job_id) = same_worker_rx.recv() => {
-                        sqlx::query_as::<_, QueuedJob>("SELECT * FROM queue WHERE id = $1")
-                        .bind(job_id)
-                        .fetch_optional(db)
-                        .await
-                        .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()))
-                    },
-                    (job, timer) = {
-                        let timer = if *METRICS_ENABLED { Some(worker_pull_duration.start_timer()) } else { None };
-                        let suspend_first = if last_checked_suspended.elapsed().as_secs() > 3 {
-                            last_checked_suspended = Instant::now();
-                            true
-                        } else { false };
-                        pull(&db, rsmq.clone(), suspend_first).map(|x| (x, timer))
-                    } => {
-                        add_time!(timing, loop_start, "post pull");
+                    }
+                    println!("received killpill for worker {}", i_worker);
+                    break
+                },
+                _ = copy_to_bucket_rx.recv() => {
+                    tracing::debug!("can_pull lock start");
+                    let _lock = CAN_PULL.write().await;
+                    // if num_workers > 1 {
+                    //     create_barrier_for_all_workers(num_workers, sync_barrier.clone()).await;
+                    // }
+                    //Arc::new(tokio::sync::Barrier::new(num_workers as usize + 1));
+                    #[cfg(feature = "enterprise")]
+                    if let Err(e) = copy_tmp_cache_to_cache().await {
+                        tracing::error!(worker = %worker_name, "failed to sync tmp cache to cache: {}", e);
+                    }
+                    tracing::debug!("can_pull lock end");
+                    Ok(None)
+                },
+                Some(job_id) = same_worker_rx.recv() => {
+                    sqlx::query_as::<_, QueuedJob>("SELECT * FROM queue WHERE id = $1")
+                    .bind(job_id)
+                    .fetch_optional(db)
+                    .await
+                    .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()))
+                },
+                (job, timer) = {
+                    let timer = if *METRICS_ENABLED { Some(worker_pull_duration.start_timer()) } else { None };
+                    let suspend_first = if last_checked_suspended.elapsed().as_secs() > 3 {
+                        last_checked_suspended = Instant::now();
+                        true
+                    } else { false };
+                    pull(&db, rsmq.clone(), suspend_first).map(|x| (x, timer))
+                } => {
+                    add_time!(timing, loop_start, "post pull");
 
-                        timer.map(|timer| {
-                            let duration_pull_s = timer.stop_and_record();
-                            worker_pull_duration_counter.inc_by(duration_pull_s);
-                        });
-                        job
+                    timer.map(|timer| {
+                        let duration_pull_s = timer.stop_and_record();
+                        worker_pull_duration_counter.inc_by(duration_pull_s);
+                    });
+                    job
 
-                    },
-                }
+                },
             }
         };
 
@@ -904,11 +1065,27 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                 jobs_executed += 1;
 
                 if let Some(dedicated_worker_tx) = dedicated_worker_tx.clone() {
+                    #[cfg(feature = "benchmark")]
+                    main_duration
+                        .fetch_add(loop_start.elapsed().as_millis() as usize, Ordering::SeqCst);
+                    #[cfg(feature = "benchmark")]
+                    let send_start = Instant::now();
+
                     if let Err(e) = dedicated_worker_tx.send(job.clone()).await {
                         tracing::info!("failed to send jobs to dedicated workers. Likely dedicated worker has been shut down. This is normal: {e:?}");
                     }
+
+                    #[cfg(feature = "benchmark")]
+                    send_duration
+                        .fetch_add(send_start.elapsed().as_millis() as usize, Ordering::SeqCst);
                     continue;
                 } else if matches!(job.job_kind, JobKind::Noop) {
+                    #[cfg(feature = "benchmark")]
+                    main_duration
+                        .fetch_add(loop_start.elapsed().as_millis() as usize, Ordering::SeqCst);
+                    #[cfg(feature = "benchmark")]
+                    let send_start = Instant::now();
+
                     job_completed_tx
                         .send(JobCompleted {
                             job,
@@ -920,6 +1097,10 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         })
                         .await
                         .expect("send job completed");
+
+                    #[cfg(feature = "benchmark")]
+                    send_duration
+                        .fetch_add(send_start.elapsed().as_millis() as usize, Ordering::SeqCst);
                 } else {
                     let token = create_token_for_owner_in_bg(&db, &job).await;
 

benchmarks/benchmark_noop.ts

@@ -1,250 +0,0 @@
-/// <reference no-default-lib="true" />
-/// <reference lib="deno.window" />
-
-import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
-import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
-import * as windmill from "https://deno.land/x/windmill@v1.38.5/mod.ts";
-import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
-import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
-export {
-  DenoLandProvider,
-  UpgradeCommand,
-} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
-
-async function login(email: string, password: string): Promise<string> {
-  return await windmill.UserService.login({
-    requestBody: {
-      email: email,
-      password: password,
-    },
-  });
-}
-
-export const VERSION = "v1.167.0";
-
-export async function main({
-  host,
-  email,
-  password,
-  token,
-  workspace,
-  jobs,
-  batches,
-}: {
-  host: string;
-  email?: string;
-  password?: string;
-  token?: string;
-  workspace: string;
-  jobs: number;
-  batches: number;
-}) {
-  windmill.setClient("", host);
-
-  console.log(
-    "Started benchmark with NOOP jobs with options",
-    JSON.stringify(
-      {
-        host,
-        email,
-        workspace,
-      },
-      null,
-      4
-    )
-  );
-
-  const config = {
-    token: "",
-    server: host,
-    workspace_id: workspace,
-  };
-
-  let final_token: string;
-  if (!token) {
-    if (email && password) {
-      final_token = await login(email, password);
-    } else {
-      console.error("Token or email with password are required.");
-      return;
-    }
-  } else {
-    final_token = token;
-  }
-
-  config.token = final_token;
-  windmill.setClient(final_token, host);
-  const enc = (s: string) => new TextEncoder().encode(s);
-
-  console.log("Disabling workers before loading jobs");
-  const disable_workers = await fetch(
-    config.server + "/api/workers/toggle?disable=true",
-    {
-      method: "GET",
-      headers: { ["Authorization"]: "Bearer " + config.token },
-    }
-  );
-  if (!disable_workers.ok) {
-    console.error(
-      "Unable to disable workers. Is the Windmill server running in benchmark mode?"
-    );
-  }
-
-  const jobsSent = jobs;
-  const batch_num = batches;
-  console.log(`Bulk creating ${jobsSent} jobs in ${batch_num} batches`);
-
-  const start_create = Date.now();
-  const all_create_operations = [];
-  for (let i = 0; i < batch_num; i++) {
-    all_create_operations.push(
-      fetch(
-        config.server +
-          "/api/w/" +
-          config.workspace_id +
-          `/jobs/add_noop_jobs/${jobsSent / batch_num}`,
-        {
-          method: "POST",
-          headers: { ["Authorization"]: "Bearer " + config.token },
-        }
-      )
-    );
-  }
-  await Promise.all(all_create_operations);
-
-  const end_create = Date.now();
-  const create_duration = end_create - start_create;
-  console.log(
-    `Jobs successfully added to the queue in ${create_duration}s. Windmill will start pulling them\n`
-  );
-  const start = Date.now();
-
-  let queue_length = jobsSent;
-  let lastElapsed = 0;
-  let lastQueueLength = queue_length;
-  const updateState = setInterval(async () => {
-    const elapsed = start ? Date.now() - start : 0;
-    queue_length = (
-      await (
-        await fetch(
-          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-          { headers: { ["Authorization"]: "Bearer " + config.token } }
-        )
-      ).json()
-    ).database_length;
-    const avgThr = (((jobsSent - queue_length) / elapsed) * 1000).toFixed(2);
-    const instThr =
-      lastElapsed > 0
-        ? (
-            ((lastQueueLength - queue_length) / (elapsed - lastElapsed)) *
-            1000
-          ).toFixed(2)
-        : 0;
-
-    lastElapsed = elapsed;
-    lastQueueLength = queue_length;
-
-    await Deno.stdout.write(
-      enc(
-        `elapsed: ${(elapsed / 1000).toFixed(2)} | jobs executed: ${
-          jobsSent - queue_length
-        }/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | queue: ${queue_length}                          \r`
-      )
-    );
-  }, 100);
-
-  console.log("Enabling workers to start processing jobs");
-  const enable_workers = await fetch(
-    config.server + "/api/workers/toggle?disable=false",
-    {
-      method: "GET",
-      headers: { ["Authorization"]: "Bearer " + config.token },
-    }
-  );
-  if (!enable_workers.ok) {
-    console.error(
-      "Unable to disable workers. Is the Windmill server running in benchmark mode?"
-    );
-  }
-
-  while (queue_length > 0) {
-    await sleep(0.1);
-  }
-
-  clearInterval(updateState);
-
-  const total_duration_sec = (Date.now() - start) / 1000.0;
-  console.log(`jobs: ${jobsSent}`);
-  console.log(`duration: ${total_duration_sec}s`);
-  console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`);
-
-  console.log(
-    "queue length:",
-    (
-      await (
-        await fetch(
-          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-          { headers: { ["Authorization"]: "Bearer " + config.token } }
-        )
-      ).json()
-    ).database_length
-  );
-  console.log("done");
-
-  return {
-    throughput: jobsSent / total_duration_sec,
-  };
-}
-
-if (import.meta.main) {
-  await new Command()
-    .name("wmillbench")
-    .description("Run Benchmark to measure throughput of windmill.")
-    .version(VERSION)
-    .option("--host <url:string>", "The windmill host to benchmark.", {
-      default: "http://127.0.0.1:8000",
-    })
-    .option("-e --email <email:string>", "The email to use to login.")
-    .option("-p --password <password:string>", "The password to use to login.")
-    .env(
-      "WM_TOKEN=<token:string>",
-      "The token to use when talking to the API server. Preferred over manual login."
-    )
-    .option(
-      "-t --token <token:string>",
-      "The token to use when talking to the API server. Preferred over manual login."
-    )
-    .env(
-      "WM_WORKSPACE=<workspace:string>",
-      "The workspace to spawn scripts from."
-    )
-    .option(
-      "-w --workspace <workspace:string>",
-      "The workspace to spawn scripts from.",
-      { default: "admins" }
-    )
-    .option("-j --jobs <jobs:number>", "Number of NOOP jobs to create.", {
-      default: 10000,
-    })
-    .option(
-      "-b --batches <batches:number>",
-      "Number of batches to create all the jobs.",
-      { default: 1 }
-    )
-    .action(main)
-    .command(
-      "upgrade",
-      new UpgradeCommand({
-        main: "main.ts",
-        args: [
-          "--allow-net",
-          "--allow-read",
-          "--allow-write",
-          "--allow-env",
-          "--unstable",
-        ],
-        provider: new DenoLandProvider({ name: "wmillbench" }),
-      })
-    )
-    .parse();
-}

benchmarks/benchmark_oneoff.ts

@@ -0,0 +1,312 @@
+/// <reference no-default-lib="true" />
+/// <reference lib="deno.window" />
+
+import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
+import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
+import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
+
+import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
+
+import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
+
+import { VERSION, createBenchScript, getFlowPayload, login } from "./lib.ts";
+
+async function verifyOutputs(uuids: string[], workspace: string) {
+  console.log("Verifying outputs");
+  let incorrectResults = 0;
+  for (const uuid of uuids) {
+    try {
+      const job = await windmill.JobService.getCompletedJob({
+        workspace,
+        id: uuid,
+      });
+      if (!job.success) {
+        console.log(`Job ${uuid} did not complete`);
+        incorrectResults++;
+      }
+      if (job.result !== uuid) {
+        console.log(`Job ${uuid} did not output the correct value`);
+        incorrectResults++;
+      }
+    } catch (_) {
+      console.log(`Job ${uuid} did not complete`);
+      incorrectResults++;
+    }
+  }
+  console.log(`Incorrect results: ${incorrectResults}`);
+}
+
+export async function main({
+  host,
+  email,
+  password,
+  token,
+  workspace,
+  kind,
+  jobs,
+  noVerify,
+}: {
+  host: string;
+  email?: string;
+  password?: string;
+  token?: string;
+  workspace: string;
+  kind: string;
+  jobs: number;
+  noVerify?: boolean;
+}) {
+  windmill.setClient("", host);
+
+  console.log(
+    "Started benchmark with options",
+    JSON.stringify(
+      {
+        host,
+        email,
+        workspace,
+        kind,
+        jobs,
+        noVerify,
+      },
+      null,
+      4
+    )
+  );
+
+  const config = {
+    token: "",
+    server: host,
+    workspace_id: workspace,
+  };
+
+  let final_token: string;
+  if (!token) {
+    if (email && password) {
+      final_token = await login(email, password);
+    } else {
+      console.error("Token or email with password are required.");
+      return;
+    }
+  } else {
+    final_token = token;
+  }
+
+  config.token = final_token;
+  windmill.setClient(final_token, host);
+  const enc = (s: string) => new TextEncoder().encode(s);
+
+  async function getQueueCount() {
+    return (
+      await (
+        await fetch(
+          config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length;
+  }
+
+  let pastJobs = 0;
+  async function getCompletedJobsCount(): Promise<number> {
+    const completedJobs = (
+      await (
+        await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/completed/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length;
+    return completedJobs - pastJobs;
+  }
+
+  if (["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)) {
+    await createBenchScript(kind, workspace);
+  }
+
+  pastJobs = await getCompletedJobsCount();
+
+  const jobsSent = jobs;
+  console.log(`Bulk creating ${jobsSent} jobs`);
+
+  const start_create = Date.now();
+  let body: string;
+  if (kind === "noop") {
+    body = JSON.stringify({
+      kind: "noop",
+    });
+  } else if (
+    ["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)
+  ) {
+    body = JSON.stringify({
+      kind: "script",
+      path: "f/benchmarks/" + kind,
+    });
+  } else if (["2steps"].includes(kind)) {
+    const payload = getFlowPayload(kind);
+    body = JSON.stringify({
+      kind: "flow",
+      flow_value: payload.value,
+    });
+  } else {
+    throw new Error("Unknown script pattern " + kind);
+  }
+
+  const response = await fetch(
+    config.server +
+      "/api/w/" +
+      config.workspace_id +
+      `/jobs/add_batch_jobs/${jobsSent}`,
+    {
+      method: "POST",
+      headers: {
+        ["Authorization"]: "Bearer " + config.token,
+        "Content-Type": "application/json",
+      },
+      body,
+    }
+  );
+  if (!response.ok) {
+    throw new Error("Failed to create jobs: " + response.statusText);
+  }
+  const uuids = await response.json();
+  const end_create = Date.now();
+  const create_duration = end_create - start_create;
+  console.log(
+    `Jobs successfully added to the queue in ${
+      create_duration / 1000
+    }s. Windmill will start pulling them\n`
+  );
+  let start = Date.now();
+
+  let completedJobs = 0;
+  let lastElapsed = 0;
+  let lastCompletedJobs = 0;
+
+  let didStart = false;
+  while (completedJobs < jobsSent) {
+    const loopStart = Date.now();
+    if (!didStart) {
+      const actual_queue = await getQueueCount();
+      if (actual_queue < jobsSent) {
+        start = Date.now();
+        didStart = true;
+      }
+    } else {
+      const elapsed = start ? Date.now() - start : 0;
+      completedJobs = await getCompletedJobsCount();
+      if (kind === "2steps") {
+        completedJobs = Math.floor(completedJobs / 3);
+      }
+      const avgThr = ((completedJobs / elapsed) * 1000).toFixed(2);
+      const instThr =
+        lastElapsed > 0
+          ? (
+              ((completedJobs - lastCompletedJobs) / (elapsed - lastElapsed)) *
+              1000
+            ).toFixed(2)
+          : 0;
+
+      lastElapsed = elapsed;
+      lastCompletedJobs = completedJobs;
+
+      await Deno.stdout.write(
+        enc(
+          `elapsed: ${(elapsed / 1000).toFixed(
+            2
+          )} | jobs executed: ${completedJobs}/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | remaining: ${
+            jobsSent - completedJobs
+          }                          \r`
+        )
+      );
+    }
+    const loopDuration = (Date.now() - loopStart) / 1000.0;
+    if (loopDuration < 0.05) {
+      await sleep(0.05 - loopDuration);
+    }
+  }
+
+  const total_duration_sec = (Date.now() - start) / 1000.0;
+
+  console.log(`\njobs: ${jobsSent}`);
+  console.log(`duration: ${total_duration_sec}s`);
+  console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`);
+
+  console.log("completed jobs", completedJobs);
+  console.log("queue length:", await getQueueCount());
+
+  if (!noVerify && kind !== "noop") {
+    await verifyOutputs(uuids, config.workspace_id);
+  }
+
+  console.log("done");
+
+  return {
+    throughput: jobsSent / total_duration_sec,
+  };
+}
+
+if (import.meta.main) {
+  await new Command()
+    .name("wmillbench")
+    .description("Run Benchmark to measure throughput of windmill.")
+    .version(VERSION)
+    .option("--host <url:string>", "The windmill host to benchmark.", {
+      default: "http://127.0.0.1:8000",
+    })
+    .option("-e --email <email:string>", "The email to use to login.", {
+      default: "admin@windmill.dev",
+    })
+    .option(
+      "-p --password <password:string>",
+      "The password to use to login.",
+      {
+        default: "changeme",
+      }
+    )
+    .env(
+      "WM_TOKEN=<token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .option(
+      "-t --token <token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .env(
+      "WM_WORKSPACE=<workspace:string>",
+      "The workspace to spawn scripts from."
+    )
+    .option(
+      "-w --workspace <workspace:string>",
+      "The workspace to spawn scripts from.",
+      { default: "admins" }
+    )
+    .option(
+      "--kind <kind:string>",
+      "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps",
+      {
+        required: true,
+      }
+    )
+    .option("-j --jobs <jobs:number>", "Number of jobs to create.", {
+      default: 10000,
+    })
+    .option("--no-verify", "Do not verify the output of the jobs.", {
+      default: false,
+    })
+    .action(main)
+    .command(
+      "upgrade",
+      new UpgradeCommand({
+        main: "main.ts",
+        args: [
+          "--allow-net",
+          "--allow-read",
+          "--allow-write",
+          "--allow-env",
+          "--unstable",
+        ],
+        provider: new DenoLandProvider({ name: "wmillbench" }),
+      })
+    )
+    .parse();
+}

benchmarks/benchmark_suite.ts

@@ -2,19 +2,46 @@ import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
 import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
 import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
 
-const VERSION = "1.167.0";
+import { main as runBenchmark } from "./benchmark_oneoff.ts";
+
+import { drawGraph, drawGraphMulti } from "./graph.ts";
+import { VERSION } from "./lib.ts";
 
 type Config = {
   benchmarks: [
     {
       graph_title: string;
-      name: string;
-      jobs: number | undefined;
-      type: "noop" | "flow" | "deno" | "python" | "go" | "bash";
+      kind: string;
+      jobs: number;
+    }
+  ];
+  extra_graphs?: [
+    {
+      graph_title: string;
+      kinds: string[];
     }
   ];
 };
 
+async function warmUp(
+  host: string,
+  email: string | undefined,
+  password: string | undefined,
+  token: string | undefined,
+  workspace: string
+) {
+  console.log("%cWarming up...", "font-weight: bold;");
+  await runBenchmark({
+    host,
+    email,
+    password,
+    token,
+    workspace,
+    kind: "noop",
+    jobs: 5000,
+  });
+}
+
 async function main({
   host,
   email,
@@ -22,7 +49,6 @@ async function main({
   token,
   workspace,
   configPath,
-  branch,
 }: {
   host: string;
   email?: string;
@@ -30,26 +56,7 @@ async function main({
   token?: string;
   workspace: string;
   configPath: string;
-  branch?: string;
 }) {
-  const { main: runNoopBenchmark } = await import(
-    branch !== undefined
-      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/benchmark_noop.ts`
-      : "./benchmark_noop.ts"
-  );
-
-  const { main: runBenchmark } = await import(
-    branch !== undefined
-      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/main.ts`
-      : "./main.ts"
-  );
-
-  const { drawGraph } = await import(
-    branch !== undefined
-      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/graph.ts`
-      : "./graph.ts"
-  );
-
   async function getConfig(configPath: string): Promise<Config> {
     if (configPath.startsWith("http")) {
       const response = await fetch(configPath);
@@ -59,56 +66,26 @@ async function main({
     }
   }
 
+  await warmUp(host, email, password, token, workspace);
+
   try {
     const config = await getConfig(configPath);
     for (const benchmark of config.benchmarks) {
       try {
         console.log(
-          "%cRunning benchmark " + benchmark.name,
+          "%cRunning benchmark " + benchmark.kind,
           "font-weight: bold;"
         );
 
-        let result:
-          | {
-              throughput: number;
-            }
-          | undefined;
-        if (benchmark.type === "noop") {
-          result = await runNoopBenchmark({
-            host,
-            email,
-            password,
-            token,
-            workspace,
-            jobs: 1000,
-            batches: 1,
-          });
-        } else {
-          result = await runBenchmark({
-            host,
-            email,
-            password,
-            token,
-            workspace,
-            workers: 1,
-            seconds: benchmark.type === "flow" ? 2 : 5,
-            metrics: "http://localhost:8001/metrics",
-            maximumThroughput: Infinity,
-            zombieTimeout: 90000,
-            histogramBuckets: [],
-            scriptPattern: [
-              "deno",
-              "python",
-              "go",
-              "bash",
-              "dedicated",
-            ].includes(benchmark.type)
-              ? benchmark.type
-              : "deno",
-            useFlows: benchmark.type === "flow",
-            hideProgress: true,
-          });
-        }
+        const result = await runBenchmark({
+          host,
+          email,
+          password,
+          token,
+          workspace,
+          kind: benchmark.kind,
+          jobs: benchmark.jobs,
+        });
 
         if (!result) {
           throw new Error("No result returned");
@@ -118,7 +95,7 @@ async function main({
           ts: Date.now(),
         };
         let data: (typeof stat)[] = [];
-        const jsonFilePath = `${benchmark.name}.json`;
+        const jsonFilePath = `${benchmark.kind}_benchmark.json`;
         try {
           const existing = await Deno.readTextFile(jsonFilePath);
           data = JSON.parse(existing);
@@ -131,15 +108,42 @@ async function main({
           data.slice(-10).map((d) => ({ ...d, date: new Date(d.ts) })),
           benchmark.graph_title
         );
-        await Deno.writeTextFile(`${benchmark.name}.svg`, svg);
+        await Deno.writeTextFile(`${benchmark.kind}_benchmark.svg`, svg);
       } catch (err) {
-        console.error("Failed to run benchmark", benchmark.name, err);
+        console.error("Failed to run benchmark", benchmark.kind, err);
       }
     }
 
+    for (const extraGraph of config.extra_graphs || []) {
+      const data: {
+        value: number;
+        ts: number;
+        date: Date;
+        kind: string;
+      }[] = [];
+      for (const kind of extraGraph.kinds) {
+        try {
+          const existing = await Deno.readTextFile(`${kind}_benchmark.json`);
+          const existingData = JSON.parse(existing)
+            .map((d: { value: number; ts: number }) => ({
+              ...d,
+              date: new Date(d.ts),
+              kind,
+            }))
+            .slice(-10);
+          data.push(...existingData);
+        } catch (err) {
+          console.log("Error while loading", kind, "benchmark data", err);
+        }
+      }
+      const svg = drawGraphMulti(data, extraGraph.graph_title);
+      await Deno.writeTextFile(`${extraGraph.kinds.join("_vs_")}.svg`, svg);
+    }
+
     Deno.exit(0); // JSDOM from drawGraph doesn't exit cleanly
   } catch (err) {
-    return console.error(`Failed to read config file ${configPath}: ${err}`);
+    console.error(`Failed to read config file ${configPath}: ${err}`);
+    Deno.exit(0); // JSDOM from drawGraph doesn't exit cleanly
   }
 }
 
@@ -150,8 +154,12 @@ await new Command()
   .option("--host <url:string>", "The windmill host to benchmark.", {
     default: "http://127.0.0.1:8000",
   })
-  .option("-e --email <email:string>", "The email to use to login.")
-  .option("-p --password <password:string>", "The password to use to login.")
+  .option("-e --email <email:string>", "The email to use to login.", {
+    default: "admin@windmill.dev",
+  })
+  .option("-p --password <password:string>", "The password to use to login.", {
+    default: "changeme",
+  })
   .env(
     "WM_TOKEN=<token:string>",
     "The token to use when talking to the API server. Preferred over manual login."
@@ -172,10 +180,6 @@ await new Command()
   .option("-c --config-path <config:string>", "The path of the config file", {
     required: true,
   })
-  .option(
-    "--branch <branch:string>",
-    "The branch to use when running remotely."
-  )
   .action(main)
   .command(
     "upgrade",

benchmarks/graph.ts

@@ -107,6 +107,167 @@ export function drawGraph(data: DataPoint[], title: string) {
   return body.node().innerHTML;
 }
 
+interface DataPointMulti extends DataPoint {
+  kind: string;
+}
+
+export function drawGraphMulti(data: DataPointMulti[], title: string) {
+  const context = {
+    jsdom: new JSDOM(""),
+  };
+  const { window } = context.jsdom;
+  const { document } = window;
+
+  const body = d3.select(document).select("body");
+
+  const width = 400;
+  const height = 200;
+
+  const marginTop = 20;
+  const marginRight = 100;
+  const marginBottom = 30;
+  const marginLeft = 60;
+
+  let svg = body
+    .append("svg")
+    .attr("xmlns", "http://www.w3.org/2000/svg")
+    .attr("width", width + marginLeft + marginRight)
+    .attr("height", height + marginTop + marginBottom);
+
+  svg
+    .append("rect")
+    .attr("width", "100%")
+    .attr("height", "100%")
+    .attr("fill", "white");
+
+  svg = svg
+    .append("g")
+    .attr("transform", "translate(" + marginLeft + "," + marginTop + ")");
+
+  const x = d3
+    .scaleTime()
+    .domain(
+      d3.extent(data, function (d: DataPoint) {
+        return d.date;
+      })
+    )
+    .nice()
+    .range([0, width]);
+
+  const xAxis = d3.axisBottom(x).ticks(5);
+
+  svg
+    .append("g")
+    .attr("transform", "translate(0," + height + ")")
+    .call(xAxis);
+
+  // Add Y axis
+  const y = d3
+    .scaleLinear()
+    .domain([
+      0,
+      d3.max(data, function (d: DataPoint) {
+        return +d.value;
+      }) * 1.5,
+    ])
+    .range([height, 0])
+    .nice();
+  svg.append("g").call(d3.axisLeft(y));
+
+  svg
+    .append("text")
+    .attr("text-anchor", "middle")
+    .attr("style", "font-size: 12px")
+    .attr("transform", "rotate(-90)")
+    .attr("y", -marginLeft + 20)
+    .attr("x", -height / 2)
+    .text("[jobs/s]");
+
+  svg
+    .append("text")
+    .attr("text-anchor", "middle")
+    .attr("style", "font-size: 16px")
+    .attr("y", 0)
+    .attr("x", width / 2)
+    .text(title);
+
+  const sumstat = d3.group(data, function (d: DataPointMulti) {
+    return d.kind;
+  });
+
+  const keys = Array.from(sumstat.keys());
+
+  const color = d3
+    .scaleOrdinal()
+    .domain(keys)
+    .range([
+      "#e41a1c",
+      "#377eb8",
+      "#4daf4a",
+      "#984ea3",
+      "#ff7f00",
+      "#ffff33",
+      "#a65628",
+      "#f781bf",
+      "#999999",
+    ]);
+
+  // Add the line
+  svg
+    .selectAll("path.line")
+    .data(sumstat)
+    .join("path")
+    .attr("class", "line")
+    .attr("fill", "none")
+    .attr("stroke", function (d) {
+      return color(d[0]);
+    })
+    .attr("stroke-width", 1.5)
+    .attr("d", (d) => {
+      return d3
+        .line()
+        .x((d) => x(d.date))
+        .y((d) => y(d.value))(d[1]);
+    });
+
+  const size = 15;
+  svg
+    .selectAll(".dot")
+    .data(keys)
+    .enter()
+    .append("rect")
+    .attr("class", "dot")
+    .attr("x", 400)
+    .attr("y", function (d, i) {
+      return 5 + i * (size + 5);
+    })
+    .attr("width", size)
+    .attr("height", size)
+    .style("fill", function (d) {
+      return color(d);
+    });
+  svg
+    .selectAll(".label")
+    .data(keys)
+    .enter()
+    .append("text")
+    .attr("class", "label")
+    .attr("x", 400 + size * 1.2)
+    .attr("y", function (d, i) {
+      return 5 + i * (size + 5) + size / 2;
+    })
+    .style("fill", function (d) {
+      return color(d);
+    })
+    .text(function (d) {
+      return d;
+    })
+    .attr("text-anchor", "left")
+    .style("alignment-baseline", "middle");
+
+  return body.node().innerHTML;
+}
+
 if (import.meta.main) {
   const svg = drawGraph(
     [
@@ -122,6 +283,33 @@ if (import.meta.main) {
     "test"
   );
 
+  const svg2 = drawGraphMulti(
+    [
+      {
+        value: 10,
+        date: new Date(86400000),
+        kind: "test",
+      },
+      {
+        value: 12,
+        date: new Date(86400000 * 2),
+        kind: "test",
+      },
+      {
+        value: 8,
+        date: new Date(86400000),
+        kind: "test2",
+      },
+      {
+        value: 9,
+        date: new Date(86400000 * 2),
+        kind: "test2",
+      },
+    ],
+    "test"
+  );
+
   console.log(svg);
+  console.log(svg2);
   Deno.exit(0);
 }

benchmarks/lib.ts

@@ -0,0 +1,257 @@
+import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
+import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
+import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
+
+export const VERSION = "v1.177.0";
+
+export async function login(email: string, password: string): Promise<string> {
+  return await windmill.UserService.login({
+    requestBody: {
+      email: email,
+      password: password,
+    },
+  });
+}
+
+async function waitForDeployment(workspace: string, hash: string) {
+  const maxTries = 20;
+  for (let i = 0; i < maxTries; i++) {
+    const resp = await windmill.ScriptService.getScriptDeploymentStatus({
+      workspace,
+      hash,
+    });
+    if (resp.lock !== null) {
+      return;
+    }
+    await sleep(0.5);
+  }
+  throw new Error("Script did not deploy in time");
+}
+
+async function waitForDedicatedWorker(workspace: string, path: string) {
+  const query = windmill.JobService.runWaitResultScriptByPath({
+    workspace,
+    path,
+    requestBody: {},
+  });
+  let timeout;
+  const timeoutPromise = new Promise((_, reject) => {
+    timeout = setTimeout(() => {
+      reject("Timeout");
+    }, 15000);
+  });
+  await Promise.race([query, timeoutPromise]);
+  clearTimeout(timeout);
+}
+
+export async function createBenchScript(
+  scriptPattern: string,
+  workspace: string
+) {
+  const path = `f/benchmarks/${scriptPattern}`;
+  const exists = await windmill.ScriptService.existsScriptByPath({
+    workspace,
+    path,
+  });
+
+  if (exists) {
+    await windmill.ScriptService.deleteScriptByPath({
+      workspace,
+      path,
+    });
+  }
+
+  let scriptContent: string;
+  let language: string;
+  let schemaProperties = {};
+  if (scriptPattern === "python") {
+    scriptContent =
+      'import os\n\ndef main():\n    return os.environ.get("WM_JOB_ID")';
+    language = "python3";
+  } else if (scriptPattern === "go") {
+    scriptContent =
+      'package inner\nimport "os"\nfunc main() (string, error) { return os.Getenv("WM_JOB_ID"), nil }';
+    language = "go";
+  } else if (scriptPattern === "bash") {
+    scriptContent = "echo $WM_JOB_ID";
+    language = "bash";
+  } else if (scriptPattern === "bun") {
+    scriptContent = 'export function main(){ return Bun.env["WM_JOB_ID"]; }';
+    language = "bun";
+  } else if (scriptPattern === "dedicated") {
+    scriptContent = "export function main(uuid){ return uuid; }";
+    language = "bun";
+    schemaProperties = {
+      uuid: { default: null, description: "", type: "string" },
+    };
+  } else if (scriptPattern === "deno") {
+    scriptContent =
+      'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
+    language = "deno";
+  } else {
+    throw new Error(
+      "Could not create script for script pattern " + scriptPattern
+    );
+  }
+
+  const hash = await windmill.ScriptService.createScript({
+    workspace,
+    requestBody: {
+      path,
+      content: scriptContent,
+      summary: scriptPattern + " benchmark",
+      description: "",
+      language: language as api.NewScript.language,
+      dedicated_worker: scriptPattern === "dedicated",
+      schema: {
+        $schema: "https://json-schema.org/draft/2020-12/schema",
+        properties: schemaProperties,
+        required: [],
+        type: "object",
+      },
+    },
+  });
+
+  await waitForDeployment(workspace, hash);
+
+  console.log("Created benchmark script at path", path);
+
+  if (scriptPattern === "dedicated") {
+    await waitForDedicatedWorker(workspace, path);
+  }
+}
+
+export const getFlowPayload = (flowPattern: string): api.FlowPreview => {
+  if (flowPattern == "branchone") {
+    return {
+      path: "branchone",
+      args: {},
+      value: {
+        modules: [
+          {
+            id: "a",
+            value: {
+              input_transforms: {},
+              language: api.RawScript.language.DENO,
+              type: "rawscript",
+              content:
+                'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
+            },
+          },
+          {
+            id: "b",
+            value: {
+              type: "branchone",
+              branches: [],
+              default: [
+                {
+                  id: "c",
+                  value: {
+                    input_transforms: {
+                      x: {
+                        type: "javascript",
+                        expr: "results.a",
+                      },
+                    },
+                    language: api.RawScript.language.DENO,
+                    type: "rawscript",
+                    content: "export function main(x: string){ return x; }",
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+    };
+  } else if (flowPattern == "branchallparrallel") {
+    return {
+      path: "branchall",
+      args: {},
+      value: {
+        modules: [
+          {
+            id: "a",
+            value: {
+              input_transforms: {},
+              language: api.RawScript.language.DENO,
+              type: "rawscript",
+              content:
+                'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
+            },
+          },
+          {
+            id: "b",
+            value: {
+              type: "branchall",
+              parallel: true,
+              branches: [
+                {
+                  modules: [
+                    {
+                      id: "c",
+                      value: {
+                        input_transforms: {
+                          x: {
+                            type: "javascript",
+                            expr: "results.a",
+                          },
+                        },
+                        language: api.RawScript.language.DENO,
+                        type: "rawscript",
+                        content: "export function main(x: string){ return x; }",
+                      },
+                    },
+                  ],
+                },
+                {
+                  modules: [
+                    {
+                      id: "d",
+                      value: {
+                        input_transforms: {
+                          x: {
+                            type: "javascript",
+                            expr: "results.a",
+                          },
+                        },
+                        language: api.RawScript.language.DENO,
+                        type: "rawscript",
+                        content: "export function main(x: string){ return x; }",
+                      },
+                    },
+                  ],
+                },
+              ],
+            },
+          },
+        ],
+      },
+    };
+  } else {
+    return {
+      path: "2steps",
+      args: {},
+      value: {
+        modules: [
+          {
+            id: "a",
+            value: {
+              input_transforms: {},
+              language: api.RawScript.language.DENO,
+              type: "rawscript",
+              content:
+                'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
+            },
+          },
+          {
+            id: "b",
+            value: {
+              type: "identity",
+            },
+          },
+        ],
+      },
+    };
+  }
+};

benchmarks/main.ts

@@ -3,11 +3,11 @@
 
 import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
 import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
-import * as windmill from "https://deno.land/x/windmill@v1.167.0/mod.ts";
-import * as api from "https://deno.land/x/windmill@v1.167.0/windmill-api/index.ts";
+import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import { Action } from "./action.ts";
 import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
 import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
+import { VERSION, createBenchScript } from "./lib.ts";
 export {
   DenoLandProvider,
   UpgradeCommand,
@@ -22,8 +22,6 @@ async function login(email: string, password: string): Promise<string> {
   });
 }
 
-export const VERSION = "v1.173.0";
-
 export async function main({
   host,
   workers: num_workers,
@@ -46,7 +44,6 @@ export async function main({
   continous,
   max,
   custom,
-  hideProgress,
 }: {
   host: string;
   workers: number;
@@ -69,7 +66,6 @@ export async function main({
   continous?: boolean;
   max?: number;
   custom?: string;
-  hideProgress?: boolean;
 }) {
   windmill.setClient("", host);
   const versionResp = await fetch(`${host}/api/version`);
@@ -130,7 +126,6 @@ export async function main({
         scriptPattern,
         zombieTimeout,
         continous,
-        hideProgress,
       },
       null,
       4
@@ -175,59 +170,16 @@ export async function main({
     scriptPattern,
     continous,
     custom: custom_content,
-    hideProgress,
   };
 
   if (
     !useFlows &&
     (scriptPattern === undefined ||
-      ["deno", "python", "go", "bash"].includes(scriptPattern))
+      ["deno", "python", "go", "bash", "bun", "dedicated"].includes(
+        scriptPattern
+      ))
   ) {
-    console.log("Creating benchmark script...");
-    const path = `f/benchmarks/${scriptPattern || "deno"}`;
-    const exists = await windmill.ScriptService.existsScriptByPath({
-      workspace,
-      path,
-    });
-
-    if (exists) {
-      await windmill.ScriptService.deleteScriptByPath({
-        workspace,
-        path,
-      });
-    }
-
-    let scriptContent: string;
-    let language: string;
-    if (scriptPattern === "python") {
-      scriptContent =
-        'import os\n\ndef main():\n    return os.environ.get("WM_JOB_ID")';
-      language = "python3";
-    } else if (scriptPattern === "go") {
-      scriptContent =
-        'package inner\nimport "os"\nfunc main() (string, error) { return os.Getenv("WM_JOB_ID"), nil }';
-      language = "go";
-    } else if (scriptPattern === "bash") {
-      scriptContent = "echo $WM_JOB_ID";
-      language = "bash";
-    } else {
-      scriptContent =
-        'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
-      language = "deno";
-    }
-
-    await windmill.ScriptService.createScript({
-      workspace,
-      requestBody: {
-        path,
-        content: scriptContent,
-        summary: (scriptPattern || "deno") + " benchmark",
-        description: "",
-        language: language as api.NewScript.language,
-      },
-    });
-
-    await sleep(5); // make sure script is created
+    await createBenchScript(scriptPattern || "deno", workspace);
   }
 
   let workers: Worker[] = new Array(num_workers);
@@ -312,13 +264,27 @@ export async function main({
   );
 
   const shutdown_start = Date.now();
-  let zombie_jobs = 0;
-  let incorrect_results = 0;
+  // let zombie_jobs = 0;
+  // let incorrect_results = 0;
+  // workers.forEach((worker, i) => {
+  //   const l = (evt: MessageEvent<any>) => {
+  //     if (evt.data.type === "zombie_jobs") {
+  //       zombie_jobs += evt.data.zombie_jobs;
+  //       incorrect_results += evt.data.incorrect_results;
+  //       worker.removeEventListener("message", l);
+  //       workers = workers.filter((w) => w != worker);
+  //       jobsSent[i] = evt.data.jobs_sent;
+  //       worker.terminate();
+  //     }
+  //   };
+  //   worker.addEventListener("message", l);
+  //   worker.postMessage(
+  //     Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
+  //   );
+  // });
   workers.forEach((worker, i) => {
     const l = (evt: MessageEvent<any>) => {
-      if (evt.data.type === "zombie_jobs") {
-        zombie_jobs += evt.data.zombie_jobs;
-        incorrect_results += evt.data.incorrect_results;
+      if (evt.data.type === "done") {
         worker.removeEventListener("message", l);
         workers = workers.filter((w) => w != worker);
         jobsSent[i] = evt.data.jobs_sent;
@@ -326,15 +292,32 @@ export async function main({
       }
     };
     worker.addEventListener("message", l);
-    worker.postMessage(
-      Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
-    );
+    worker.postMessage("done");
   });
 
   console.log("waiting for shutdown\n");
   while (workers.length > 0) {
     await sleep(0.1);
   }
+
+  let queue_length = await getQueueCount();
+  const updateQueue = setInterval(async () => {
+    queue_length = (
+      await (
+        await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length;
+    await Deno.stdout.write(enc(`queue length: ${queue_length}\r`));
+  }, 100);
+  while (queue_length > 0) {
+    await sleep(0.1);
+  }
+
+  clearInterval(updateQueue);
+
   sum = jobsSent.reduce((a, b) => a + b, 0);
 
   const tts = (Date.now() - shutdown_start) / 1000;
@@ -344,8 +327,8 @@ export async function main({
   console.log("time (s + tts):", time);
   console.log("throughput /s (jobs/time):", sum / time);
 
-  console.log("zombie jobs: ", zombie_jobs);
-  console.log("incorrect results: ", incorrect_results);
+  // console.log("zombie jobs: ", zombie_jobs);
+  // console.log("incorrect results: ", incorrect_results);
   console.log(
     "queue length:",
     (
@@ -495,7 +478,7 @@ if (import.meta.main) {
     )
     .option(
       "--script-pattern <pattern:string>",
-      "Use a different script pattern among: deno, identity, python, go, bash (Default deno)"
+      "Use a different script pattern among: deno, identity, python, go, bash, dedicated, bun (Default deno)"
     )
     .option("--custom <custom_path:string>", "Use custom actions during bench")
     .option(

benchmarks/suite_config.json

@@ -1,34 +1,64 @@
 {
   "benchmarks": [
     {
-      "name": "noop_benchmark",
       "graph_title": "noop throughput benchmark (single worker)",
-      "type": "noop"
+      "kind": "noop",
+      "jobs": 30000
     },
     {
-      "name": "flow_benchmark",
       "graph_title": "flow throughput benchmark (single worker)",
-      "type": "flow"
+      "kind": "2steps",
+      "jobs": 250
+    },
+    {
+      "graph_title": "dedicated throughput benchmark (single worker)",
+      "kind": "dedicated",
+      "jobs": 30000
     },
     {
-      "name": "deno_benchmark",
       "graph_title": "deno throughput benchmark (single worker)",
-      "type": "deno"
+      "kind": "deno",
+      "jobs": 500
+    },
+    {
+      "graph_title": "bun throughput benchmark (single worker)",
+      "kind": "bun",
+      "jobs": 500
     },
     {
-      "name": "python_benchmark",
       "graph_title": "python throughput benchmark (single worker)",
-      "type": "python"
+      "kind": "python",
+      "jobs": 500
     },
     {
-      "name": "go_benchmark",
       "graph_title": "go throughput benchmark (single worker)",
-      "type": "go"
+      "kind": "go",
+      "jobs": 500
     },
     {
-      "name": "bash_benchmark",
       "graph_title": "bash throughput benchmark (single worker)",
-      "type": "bash"
+      "kind": "bash",
+      "jobs": 500
+    }
+  ],
+  "extra_graphs": [
+    {
+      "graph_title": "go vs python vs deno vs bun vs bash",
+      "kinds": [
+        "go",
+        "python",
+        "deno",
+        "bun",
+        "bash"
+      ]
+    },
+    {
+      "graph_title": "bun vs dedicated vs noop",
+      "kinds": [
+        "bun",
+        "dedicated",
+        "noop"
+      ]
     }
   ]
 }

benchmarks/worker.ts

@@ -1,10 +1,21 @@
 /// <reference no-default-lib="true" />
 /// <reference lib="deno.worker" />
 import { sleep } from "https://deno.land/x/sleep@v1.2.1/sleep.ts";
-import * as windmill from "https://deno.land/x/windmill@v1.167.0/mod.ts";
-import * as api from "https://deno.land/x/windmill@v1.167.0/windmill-api/index.ts";
-import { Job } from "https://deno.land/x/windmill@v1.167.0/windmill-api/index.ts";
+import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
+import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 import { Action, evaluate } from "./action.ts";
+import { getFlowPayload } from "./lib.ts";
+
+async function getQueueCount() {
+  return (
+    await (
+      await fetch(
+        config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+        { headers: { ["Authorization"]: "Bearer " + config.token } }
+      )
+    ).json()
+  ).database_length;
+}
 
 const promise = new Promise<{
   workspace_id: string;
@@ -46,11 +57,11 @@ let cont = true;
 let total_spawned = 0;
 
 const start_time: number = Date.now();
-let complete_timeout = Infinity;
+// let complete_timeout = Infinity;
 
 self.onmessage = (evt) => {
   cont = false;
-  complete_timeout = evt.data;
+  // complete_timeout = evt.data;
 };
 
 const updateStatusInterval = setInterval(() => {
@@ -85,141 +96,8 @@ while (cont) {
       await evaluate(config.custom);
       continue;
     } else if (config.useFlows) {
-      let payload: api.FlowPreview;
-      if (config.flowPattern == "branchone") {
-        payload = {
-          path: "branchone",
-          args: {},
-          value: {
-            modules: [
-              {
-                id: "a",
-                value: {
-                  input_transforms: {},
-                  language: api.RawScript.language.DENO,
-                  type: "rawscript",
-                  content:
-                    'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
-                },
-              },
-              {
-                id: "b",
-                value: {
-                  type: "branchone",
-                  branches: [],
-                  default: [
-                    {
-                      id: "c",
-                      value: {
-                        input_transforms: {
-                          x: {
-                            type: "javascript",
-                            expr: "results.a",
-                          },
-                        },
-                        language: api.RawScript.language.DENO,
-                        type: "rawscript",
-                        content: "export function main(x: string){ return x; }",
-                      },
-                    },
-                  ],
-                },
-              },
-            ],
-          },
-        };
-      } else if (config.flowPattern == "branchallparrallel") {
-        payload = {
-          path: "branchall",
-          args: {},
-          value: {
-            modules: [
-              {
-                id: "a",
-                value: {
-                  input_transforms: {},
-                  language: api.RawScript.language.DENO,
-                  type: "rawscript",
-                  content:
-                    'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
-                },
-              },
-              {
-                id: "b",
-                value: {
-                  type: "branchall",
-                  parallel: true,
-                  branches: [
-                    {
-                      modules: [
-                        {
-                          id: "c",
-                          value: {
-                            input_transforms: {
-                              x: {
-                                type: "javascript",
-                                expr: "results.a",
-                              },
-                            },
-                            language: api.RawScript.language.DENO,
-                            type: "rawscript",
-                            content:
-                              "export function main(x: string){ return x; }",
-                          },
-                        },
-                      ],
-                    },
-                    {
-                      modules: [
-                        {
-                          id: "d",
-                          value: {
-                            input_transforms: {
-                              x: {
-                                type: "javascript",
-                                expr: "results.a",
-                              },
-                            },
-                            language: api.RawScript.language.DENO,
-                            type: "rawscript",
-                            content:
-                              "export function main(x: string){ return x; }",
-                          },
-                        },
-                      ],
-                    },
-                  ],
-                },
-              },
-            ],
-          },
-        };
-      } else {
-        payload = {
-          path: "2steps",
-          args: {},
-          value: {
-            modules: [
-              {
-                id: "a",
-                value: {
-                  input_transforms: {},
-                  language: api.RawScript.language.DENO,
-                  type: "rawscript",
-                  content:
-                    'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
-                },
-              },
-              {
-                id: "b",
-                value: {
-                  type: "identity",
-                },
-              },
-            ],
-          },
-        };
-      }
+      const payload = getFlowPayload(config.flowPattern);
+
       uuid = await windmill.JobService.runFlowPreview({
         workspace: config.workspace_id,
         requestBody: payload,
@@ -261,99 +139,97 @@ while (cont) {
 
 clearInterval(updateStatusInterval);
 
-const end_time = Date.now() + complete_timeout;
+// const end_time = Date.now() + complete_timeout;
 
-let incorrect_results = 0;
-const enc = (s: string) => new TextEncoder().encode(s);
+// let incorrect_results = 0;
+// const enc = (s: string) => new TextEncoder().encode(s);
 
-async function getQueueCount() {
-  return (
-    await (
-      await fetch(
-        config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-        { headers: { ["Authorization"]: "Bearer " + config.token } }
-      )
-    ).json()
-  ).database_length;
-}
+// let last_queue_length = await getQueueCount();
+// console.log(`waiting for ${last_queue_length} jobs to complete...`);
 
-let last_queue_length = await getQueueCount();
-console.log(`waiting for ${last_queue_length} jobs to complete...`);
+// while (
+//   outstanding.length > 0 &&
+//   last_queue_length > 0 &&
+//   Date.now() < end_time
+// ) {
+//   try {
+//     if (!config.hideProgress) {
+//       await Deno.stdout.write(
+//         enc(
+//           "\rwaiting for jobs to complete: outstanding " +
+//             outstanding.length +
+//             " - queue" +
+//             last_queue_length +
+//             "\n"
+//         )
+//       );
+//     }
+//     last_queue_length = await getQueueCount();
 
-while (
-  outstanding.length > 0 &&
-  last_queue_length > 0 &&
-  Date.now() < end_time
-) {
-  try {
-    if (!config.hideProgress) {
-      await Deno.stdout.write(
-        enc(
-          "\rwaiting for jobs to complete: outstanding " +
-            outstanding.length +
-            " - queue" +
-            last_queue_length +
-            "\n"
-        )
-      );
-    }
-    last_queue_length = await getQueueCount();
+//     const uuid = outstanding.shift()!;
 
-    const uuid = outstanding.shift()!;
+//     let r: Job;
+//     try {
+//       r = await windmill.JobService.getJob({
+//         workspace: config.workspace_id,
+//         id: uuid,
+//       });
+//     } catch (e) {
+//       console.log("job not found: " + uuid + " " + e.message);
+//       continue;
+//     }
+//     if (r.type == "QueuedJob") {
+//       outstanding.push(uuid);
 
-    let r: Job;
-    try {
-      r = await windmill.JobService.getJob({
-        workspace: config.workspace_id,
-        id: uuid,
-      });
-    } catch (e) {
-      console.log("job not found: " + uuid + " " + e.message);
-      continue;
-    }
-    if (r.type == "QueuedJob") {
-      outstanding.push(uuid);
+//       if (!config.hideProgress) {
+//         await Deno.stdout.write(
+//           enc(`uuid: ${uuid}, queue length: ${last_queue_length}\r`)
+//         );
+//       }
+//     } else {
+//       r = r as api.CompletedJob;
+//       try {
+//         if (
+//           ![
+//             "httpversion",
+//             "identity",
+//             "httpslow",
+//             "noop",
+//             "dedicated",
+//           ].includes(config.scriptPattern) &&
+//           r.result != uuid
+//         ) {
+//           console.log(
+//             "job did not return correct UUID: " +
+//               r.result +
+//               " != " +
+//               uuid +
+//               "job: \n" +
+//               JSON.stringify(r, null, 2)
+//           );
+//           incorrect_results++;
+//         } else {
+//           // console.log(r.result);
+//         }
+//       } catch (e) {
+//         console.log("error during wait: ", e);
+//         outstanding.push(uuid);
+//       }
+//     }
+//   } catch (e) {
+//     console.log("error while waiting for outstanding jobs, sleeing: ", e);
+//     await sleep(0.5);
+//   }
+// }
 
-      if (!config.hideProgress) {
-        await Deno.stdout.write(
-          enc(`uuid: ${uuid}, queue length: ${last_queue_length}\r`)
-        );
-      }
-    } else {
-      r = r as api.CompletedJob;
-      try {
-        if (
-          !["httpversion", "identity", "httpslow", "noop"].includes(
-            config.scriptPattern
-          ) &&
-          r.result != uuid
-        ) {
-          console.log(
-            "job did not return correct UUID: " +
-              r.result +
-              " != " +
-              uuid +
-              "job: \n" +
-              JSON.stringify(r, null, 2)
-          );
-          incorrect_results++;
-        } else {
-          // console.log(r.result);
-        }
-      } catch (e) {
-        console.log("error during wait: ", e);
-        outstanding.push(uuid);
-      }
-    }
-  } catch (e) {
-    console.log("error while waiting for outstanding jobs, sleeing: ", e);
-    await sleep(0.5);
-  }
-}
+// self.postMessage({
+//   type: "zombie_jobs",
+//   zombie_jobs: outstanding.length,
+//   incorrect_results,
+//   jobs_sent: total_spawned,
+// });
 
 self.postMessage({
-  type: "zombie_jobs",
-  zombie_jobs: outstanding.length,
-  incorrect_results,
+  type: "done",
   jobs_sent: total_spawned,
 });

cli/main.ts

@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
   }
 });
 
-export const VERSION = "v1.173.0";
+export const VERSION = "v1.177.0";
 
 let command: any = new Command()
   .name("wmill")

docker-compose.yml

@@ -12,7 +12,7 @@ services:
     expose:
       - 5432
     environment:
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_PASSWORD: changeme
       POSTGRES_DB: windmill
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -30,37 +30,34 @@ services:
       - 8000
     environment:
       - DATABASE_URL=${DATABASE_URL}
-      - BASE_URL=${WM_BASE_URL}
       - RUST_LOG=info
       ## You can set the number of workers to 1 and not need any separate worker service but not recommended
       - NUM_WORKERS=0
       - DISABLE_SERVER=false
       - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
-      - REQUEST_SIZE_LIMIT=${WM_REQUEST_SIZE_LIMIT}
       # LICENSE_KEY is only needed for the enterprise edition
       - LICENSE_KEY=${WM_LICENSE_KEY}
     depends_on:
       db:
         condition: service_healthy
-    volumes:
-      # See Oauth (https://docs.windmill.dev/docs/misc/setup_oauth)
-      - ./oauth.json:/usr/src/app/oauth.json
 
   windmill_worker:
     image: ${WM_IMAGE}
     pull_policy: always
     deploy:
       replicas: 3
+      resources:
+        limits:
+          cpus: "1"
+          memory: 2048M
     restart: unless-stopped
     environment:
       - DATABASE_URL=${DATABASE_URL}
-      - BASE_URL=${WM_BASE_URL}
       - RUST_LOG=info
       - DISABLE_SERVER=true
       - KEEP_JOB_DIR=false
       - METRICS_ADDR=false
-      # To handle all tags, remove the env variable altogether. If you do so, you can remove the windmill_worker_native containers.
-      - WORKER_TAGS=deno,python3,go,bash,powershell,dependency,flow,hub,other,bun
+      - WORKER_GROUP=default
       # LICENSE_KEY is only needed for the enterprise edition
       - LICENSE_KEY=${WM_LICENSE_KEY}
     depends_on:
@@ -70,41 +67,29 @@ services:
     volumes:
       # mount the docker socket to allow to run docker containers from within the workers
       - /var/run/docker.sock:/var/run/docker.sock
-      # See Oauth (https://docs.windmill.dev/docs/misc/setup_oauth)
-      - ./oauth.json:/usr/src/app/oauth.json
       - worker_dependency_cache:/tmp/windmill/cache
 
-  ## This worker is specialized for "native" jobs. They run in-process and can thus be parallelized to more than 1 at a time on a given worker which is why NUM_WORKERS is set to 4
+  ## This worker is specialized for "native" jobs. Native jobs run in-process and thus are much more lightweight than other jobs
   windmill_worker_native:
     # Use ghcr.io/windmill-labs/windmill-ee:main for the ee
     image: ${WM_IMAGE}
     pull_policy: always
     deploy:
-      replicas: 1
+      replicas: 2
       resources:
-          limits:
-            cpus: "0.25"
-            memory: 512M
+        limits:
+          cpus: "0.1"
+          memory: 128M
     restart: unless-stopped
     environment:
       - DATABASE_URL=${DATABASE_URL}
-      - BASE_URL=${WM_BASE_URL}
       - RUST_LOG=info
       - DISABLE_SERVER=true
-      - KEEP_JOB_DIR=false
       - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
-      - NUM_WORKERS=4
-      - WORKER_TAGS=nativets,postgresql,mysql,graphql,snowflake
       - WORKER_GROUP=native
-      # LICENSE_KEY is only needed for the enterprise edition
-      # - LICENSE_KEY=${WM_LICENSE_KEY}
     depends_on:
       db:
         condition: service_healthy
-    volumes:
-      # See Oauth (https://docs.windmill.dev/docs/misc/setup_oauth)
-      - ./oauth.json:/usr/src/app/oauth.json
-
 
   lsp:
     image: ghcr.io/windmill-labs/windmill-lsp:latest

docker/DockerfilePuppeteer

@@ -0,0 +1,14 @@
+FROM ghcr.io/windmill-labs/windmill-ee:main
+
+
+RUN apt update
+RUN apt install npm -y
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key |  gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update -y
+RUN apt install nodejs libnss3-dev  libatk1.0-0 libatk-bridge2.0-0 libcups2-dev  libdrm-dev libxkbcommon-dev libxcomposite-dev libxdamage-dev libxrandr-dev\
+  libgbm-dev libpango-1.0 libcairo-dev libasound-dev -y
+RUN npm install -g puppeteer -y
+
+CMD ["windmill"]

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.173.0",
+	"version": "1.177.0",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.173.0",
+			"version": "1.177.0",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.173.0",
+	"version": "1.177.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",

frontend/src/lib/components/AppConnect.svelte

@@ -57,12 +57,10 @@
 
 <script lang="ts">
 	import { oauthStore, workspaceStore } from '$lib/stores'
-	import { faMinus, faPlus } from '@fortawesome/free-solid-svg-icons'
 	import IconedResourceType from './IconedResourceType.svelte'
 	import { OauthService, ResourceService, VariableService, type TokenResponse } from '$lib/gen'
 	import { emptyString, truncateRev } from '$lib/utils'
 	import { createEventDispatcher } from 'svelte'
-	import Icon from 'svelte-awesome'
 	import Path from './Path.svelte'
 	import { Button, Drawer, Skeleton } from './common'
 	import DrawerContent from './common/drawer/DrawerContent.svelte'
@@ -71,10 +69,11 @@
 	import autosize from 'svelte-autosize'
 	import WhitelistIp from './WhitelistIp.svelte'
 	import { sendUserToast } from '$lib/toast'
+	import OauthScopes from './OauthScopes.svelte'
 
 	export let newPageOAuth = false
 
-	const nativeLanguagesCategory = ['postgresql', 'mysql', 'bigquery', 'snowflake', 'graphql'];
+	const nativeLanguagesCategory = ['postgresql', 'mysql', 'bigquery', 'snowflake', 'graphql']
 
 	let filter = ''
 	let manual = false
@@ -195,14 +194,14 @@
 					linkedSecret: undefined
 				}
 			])
-			const filteredNativeLanguages = filteredConnectsManual?.filter(([key, _]) =>
-      nativeLanguagesCategory.includes(key)
-    );
+		const filteredNativeLanguages = filteredConnectsManual?.filter(([key, _]) =>
+			nativeLanguagesCategory.includes(key)
+		)
 
-    filteredConnectsManual = [
-      ...(filteredNativeLanguages ?? []),
-      ...(filteredConnectsManual ?? []).filter(([key, _]) => !nativeLanguagesCategory.includes(key))
-    ];
+		filteredConnectsManual = [
+			...(filteredNativeLanguages ?? []),
+			...(filteredConnectsManual ?? []).filter(([key, _]) => !nativeLanguagesCategory.includes(key))
+		]
 	}
 
 	async function next() {
@@ -397,93 +396,62 @@
 			{#if manual == false && resource_type != ''}
 				<h3>Scopes</h3>
 				{#if !manual && resource_type != ''}
-					{#each scopes as v}
-						<div class="flex flex-row max-w-md mb-2">
-							<input type="text" bind:value={v} />
-							<Button
-								variant="border"
-								color="red"
-								size="xs"
-								btnClasses="mx-6"
-								on:click={() => {
-									scopes = scopes.filter((el) => el != v)
-								}}
-							>
-								<Icon data={faMinus} />
-							</Button>
-						</div>
-					{/each}
-					<div class="flex items-center mt-1">
-						<Button
-							variant="border"
-							color="blue"
-							hover="yo"
-							size="sm"
-							endIcon={{ icon: faPlus }}
-							on:click={() => {
-								scopes = scopes.concat('')
-							}}
-						>
-							Add item
-						</Button>
-						<span class="ml-2 text-sm text-tertiary">
-							({(scopes ?? []).length} item{(scopes ?? []).length > 1 ? 's' : ''})
-						</span>
-					</div>
+					<OauthScopes bind:scopes />
 				{/if}
 			{/if}
 
 			<h2 class="mt-8 mb-4">Others</h2>
-      <div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
-        {#if filteredConnectsManual}
-          {#each filteredConnectsManual as [key, _]}
-            {#if nativeLanguagesCategory.includes(key)}
-              <Button
-                size="sm"
-                variant="border"
-                color={key === resource_type ? 'blue' : 'light'}
-                btnClasses={key === resource_type ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
-                on:click={() => {
-                  manual = true;
-                  resource_type = key;
-                  next();
-                  dispatch('click');
-                }}
-              >
-                <IconedResourceType name={key} after={true} width="20px" height="20px" />
-              </Button>
-            {/if}
-          {/each}
-        {/if}
-      </div>
+			<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
+				{#if filteredConnectsManual}
+					{#each filteredConnectsManual as [key, _]}
+						{#if nativeLanguagesCategory.includes(key)}
+							<Button
+								size="sm"
+								variant="border"
+								color={key === resource_type ? 'blue' : 'light'}
+								btnClasses={key === resource_type ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
+								on:click={() => {
+									manual = true
+									resource_type = key
+									next()
+									dispatch('click')
+								}}
+							>
+								<IconedResourceType name={key} after={true} width="20px" height="20px" />
+							</Button>
+						{/if}
+					{/each}
+				{/if}
+			</div>
 
-	  <h2 class="mt-8 mb-4"></h2>
-	  <div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
-		  {#if filteredConnectsManual}
-			  {#each filteredConnectsManual as [key, _]}
-				  {#if !nativeLanguagesCategory.includes(key)} <!-- Exclude specific items -->
-					  <Button
-						  size="sm"
-						  variant="border"
-						  color={key === resource_type ? 'blue' : 'light'}
-						  btnClasses={key === resource_type ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
-						  on:click={() => {
-							  manual = true;
-							  resource_type = key;
-							  next();
-							  dispatch('click');
-						  }}
-					  >
-						  <IconedResourceType name={key} after={true} width="20px" height="20px" />
-					  </Button>
-				  {/if}
-			  {/each}
-		  {:else}
-			  {#each new Array(9) as _}
-				  <Skeleton layout={[[2]]} />
-			  {/each}
-		  {/if}
-	  </div>	  
+			<h2 class="mt-8 mb-4" />
+			<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
+				{#if filteredConnectsManual}
+					{#each filteredConnectsManual as [key, _]}
+						{#if !nativeLanguagesCategory.includes(key)}
+							<!-- Exclude specific items -->
+							<Button
+								size="sm"
+								variant="border"
+								color={key === resource_type ? 'blue' : 'light'}
+								btnClasses={key === resource_type ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
+								on:click={() => {
+									manual = true
+									resource_type = key
+									next()
+									dispatch('click')
+								}}
+							>
+								<IconedResourceType name={key} after={true} width="20px" height="20px" />
+							</Button>
+						{/if}
+					{/each}
+				{:else}
+					{#each new Array(9) as _}
+						<Skeleton layout={[[2]]} />
+					{/each}
+				{/if}
+			</div>
 		{:else if step == 2 && manual}
 			<Path
 				bind:error={pathError}

frontend/src/lib/components/ArgInput.svelte

@@ -114,7 +114,10 @@
 	}
 
 	function evalValueToRaw() {
-		rawValue = inputCat === 'object' ? JSON.stringify(value, null, 2) : undefined
+		rawValue =
+			inputCat === 'object' || inputCat === 'resource-object'
+				? JSON.stringify(value, null, 2)
+				: undefined
 	}
 
 	evalValueToRaw()
@@ -351,6 +354,8 @@
 						</Button>
 					</div>
 				</div>
+			{:else if inputCat == 'resource-object' && resourceTypes == undefined}
+				<span class="text-2xs text-tertiary">Loading resource types...</span>
 			{:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format.split('-').length > 1 && resourceTypes.includes(format.substring('resource-'.length))))}
 				<ObjectResourceInput {disablePortal} {format} bind:value {showSchemaExplorer} />
 			{:else if inputCat == 'object' || inputCat == 'resource-object'}

frontend/src/lib/components/CustomOauth.svelte

@@ -0,0 +1,76 @@
+<script lang="ts">
+	import OauthExtraParams from './OauthExtraParams.svelte'
+	import OauthScopes from './OauthScopes.svelte'
+	import Toggle from './Toggle.svelte'
+	import Tooltip from './Tooltip.svelte'
+
+	export let connect_config = {
+		scopes: [],
+		auth_url: '',
+		token_url: '',
+		req_body_auth: false,
+		extra_params: {},
+		extra_params_callback: {}
+	}
+
+	$: if (!connect_config) {
+		connect_config = {
+			scopes: [],
+			auth_url: '',
+			token_url: '',
+			req_body_auth: false,
+			extra_params: {},
+			extra_params_callback: {}
+		}
+	}
+</script>
+
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm">Auth URL</span>
+	<input
+		type="text"
+		placeholder="https://github.com/login/oauth/authorize"
+		bind:value={connect_config.auth_url}
+	/>
+</label>
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm">Token URL</span>
+	<input
+		type="text"
+		placeholder="https://github.com/login/oauth/access_token"
+		bind:value={connect_config.token_url}
+	/>
+</label>
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm">Scopes</span>
+	<OauthScopes bind:scopes={connect_config.scopes} />
+</label>
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm"
+		>Extra Query Args for Authorize Request&nbsp;<Tooltip
+			>Not needed in most cases. Examples of uses: google apis require the 2 extra args
+			"access_type=offline&prompt=consent"</Tooltip
+		></span
+	>
+	<OauthExtraParams bind:extra_params={connect_config.extra_params} />
+</label>
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm"
+		>Extra Query Args for Token request <Tooltip>Not needed in most cases</Tooltip></span
+	>
+	<OauthExtraParams bind:extra_params={connect_config.extra_params_callback} />
+</label>
+<label class="block pb-2">
+	<span class="text-primary font-semibold text-sm"
+		>Payload <Tooltip
+			>Auth is passed in query most commonly. LinkedIn is an example of OAuth using
+			x-www-form-urlencoded
+		</Tooltip></span
+	>
+	<div>
+		<Toggle
+			options={{ left: 'in query args', right: 'in body x-www-form-urlencoded' }}
+			bind:checked={connect_config.req_body_auth}
+		/></div
+	>
+</label>

frontend/src/lib/components/DBSchemaExplorer.svelte

@@ -14,7 +14,7 @@
 	import { sendUserToast, tryEvery } from '$lib/utils'
 	import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
 	import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
-	import { buildClientSchema, printSchema } from 'graphql'
+	import { buildClientSchema, getIntrospectionQuery, printSchema } from 'graphql'
 	import GraphqlSchemaViewer from './GraphqlSchemaViewer.svelte'
 	import { faRefresh } from '@fortawesome/free-solid-svg-icons'
 
@@ -25,214 +25,147 @@
 
 	let drawer: Drawer | undefined
 
-	const scripts: {
-		[key: string]: {
+	const scripts: Record<
+		string,
+		{
 			code: string
 			lang: string
+			processingFn?: (any: any) => SQLSchema['schema']
+			argName: string
 		}
-	} = {
+	> = {
 		postgresql: {
-			lang: 'deno',
-			code: `import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
-export async function main(args: any) {
-  // Create a new client with the provided connection details
-  const u = new URL("postgres://")
-	u.hash = ''
-	u.search = '?sslmode=' + args.sslmode
-	u.pathname = args.dbname
-	u.host = args.host
-	u.port = args.port
-	u.password = args.password
-	u.username = args.user
-  const client = new Client(u.toString())
-  // Connect to the postgres database
-  await client.connect();
-  const result = await client.queryObject(\`SELECT 
-      table_name, 
-      column_name, 
-      udt_name,
-      column_default,
-      is_nullable,
-      table_schema
-    FROM 
-      information_schema.columns
-    WHERE table_schema != 'pg_catalog' AND 
-      table_schema != 'information_schema'\`);
-  const schemas = result.rows.reduce((acc, a) => {
-    const table_schema = a.table_schema;
-    delete a.table_schema;
-    acc[table_schema] = acc[table_schema] || [];
-    acc[table_schema].push(a);
-    return acc;
-  }, {});
-  const data = {};
-  for (const key in schemas) {
-    data[key] = schemas[key].reduce((acc, a) => {
-      const table_name = a.table_name;
-      delete a.table_name;
-      acc[table_name] = acc[table_name] || {};
-      const p = {
-        type: a.udt_name,
-        required: a.is_nullable === "NO",
-      }
-      if (a.column_default) {
-        p.default = a.column_default
-      }
-      acc[table_name][a.column_name] = p;
-      return acc;
-    }, {});
-  }
-  return data;
-}`
+			code: `SELECT table_name, column_name, udt_name, column_default, is_nullable, table_schema FROM information_schema.columns WHERE table_schema != 'pg_catalog' AND table_schema != 'information_schema'`,
+			processingFn: (rows) => {
+				const schemas = rows.reduce((acc, a) => {
+					const table_schema = a.table_schema
+					delete a.table_schema
+					acc[table_schema] = acc[table_schema] || []
+					acc[table_schema].push(a)
+					return acc
+				}, {})
+				const data = {}
+				for (const key in schemas) {
+					data[key] = schemas[key].reduce((acc, a) => {
+						const table_name = a.table_name
+						delete a.table_name
+						acc[table_name] = acc[table_name] || {}
+						const p: {
+							type: string
+							required: boolean
+							default?: string
+						} = {
+							type: a.udt_name,
+							required: a.is_nullable === 'NO'
+						}
+						if (a.column_default) {
+							p.default = a.column_default
+						}
+						acc[table_name][a.column_name] = p
+						return acc
+					}, {})
+				}
+				return data
+			},
+			lang: 'postgresql',
+			argName: 'database'
 		},
 		mysql: {
-			code: `import { Client } from "https://deno.land/x/mysql@v2.11.0/mod.ts";
-export async function main(args: any) {
-  const conn = await new Client().connect({
-    hostname: args.host,
-    port: args.port,
-    username: args.user,
-    db: args.database,
-    password: args.password,
-  });
-  const result = await conn.execute(
-    "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
-  );
-  const schemas = result.rows.reduce((acc, a) => {
-    const table_schema = a.TABLE_SCHEMA;
-    delete a.TABLE_SCHEMA;
-    acc[table_schema] = acc[table_schema] || [];
-    acc[table_schema].push(a);
-    return acc;
-  }, {});
-  const data = {};
-  for (const key in schemas) {
-    data[key] = schemas[key].reduce((acc, a) => {
-      const table_name = a.TABLE_NAME;
-      delete a.TABLE_NAME;
-      acc[table_name] = acc[table_name] || {};
-      const p = {
-        type: a.DATA_TYPE,
-        required: a.is_nullable === "NO",
-      };
-      if (a.column_default) {
-        p.default = a.COLUMN_DEFAULT;
-      }
-      acc[table_name][a.COLUMN_NAME] = p;
-      return acc;
-    }, {});
-  }
-  return data;
-}`,
-			lang: 'deno'
+			code: "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
+			processingFn: (rows) => {
+				const schemas = rows.reduce((acc, a) => {
+					const table_schema = a.TABLE_SCHEMA
+					delete a.TABLE_SCHEMA
+					acc[table_schema] = acc[table_schema] || []
+					acc[table_schema].push(a)
+					return acc
+				}, {})
+				const data = {}
+				for (const key in schemas) {
+					data[key] = schemas[key].reduce((acc, a) => {
+						const table_name = a.TABLE_NAME
+						delete a.TABLE_NAME
+						acc[table_name] = acc[table_name] || {}
+						const p: {
+							type: string
+							required: boolean
+							default?: string
+						} = {
+							type: a.DATA_TYPE,
+							required: a.is_nullable === 'NO'
+						}
+						if (a.column_default) {
+							p.default = a.COLUMN_DEFAULT
+						}
+						acc[table_name][a.COLUMN_NAME] = p
+						return acc
+					}, {})
+				}
+				return data
+			},
+			lang: 'mysql',
+			argName: 'database'
 		},
 		graphql: {
-			code: `import { getIntrospectionQuery } from "npm:graphql@16.7.1";
-export async function main(args: any) {
-  const headers: { [key: string]: string } = {
-    "Content-Type": "application/json",
-  };
-  if (args.bearer_token) {
-    headers["authorization"] = "Bearer " + args.bearer_token;
-  }
-  const response = await fetch(args.base_url, {
-    method: "POST",
-    headers,
-    body: JSON.stringify({
-      query: getIntrospectionQuery(),
-    }),
-  });
-  if (!response.ok) {
-    throw new Error("Could not query schema");
-  }
-  const schema = (await response.json()).data;
-  return schema;
-}`,
-			lang: 'deno'
+			code: getIntrospectionQuery(),
+			lang: 'graphql',
+			argName: 'api'
 		},
 		bigquery: {
-			code: `
-#requirements:
-#google-cloud-bigquery==3.11.4
-from google.cloud import bigquery as bq
-from google.oauth2 import service_account
-def main(args):
-    credentials = service_account.Credentials.from_service_account_info(args)
-    client = bq.Client(credentials=credentials)
-    datasets = list(client.list_datasets())  # Make an API request.
-    schema = dict()
-    for dataset in datasets:
-        schema[dataset.dataset_id] = dict()
-        query = f"""
-SELECT 
-  table_name, ARRAY_AGG(STRUCT( 
-      if(is_nullable = 'YES', true, false) AS required,
-      column_name AS name,
-      data_type AS type,
-      if(column_default = 'NULL', null, column_default) AS \`default\`)
-    ORDER BY ordinal_position) AS schema
-FROM
-  \`{dataset.dataset_id}\`.INFORMATION_SCHEMA.COLUMNS
-GROUP BY
-  table_name
-"""
-        query_job = client.query(query)  # API request
-        rows = query_job.result()
-        for row in rows:
-            cols = []
-            for col in row[1]:
-                if col['default'] is None:
-                    del col['default']
-                cols.append(col)
-            schema[dataset.dataset_id][row[0]] = cols
-    return schema
-`,
-			lang: 'python3'
+			code: `import { BigQuery } from '@google-cloud/bigquery@7.2.0';
+export async function main(args: bigquery) {
+  const bq = new BigQuery({
+    credentials: args
+  })
+  const [datasets] = await bq.getDatasets();
+  const schema = {}
+  for (const dataset of datasets) {
+    schema[dataset.id] = {}
+    const query = "SELECT table_name, ARRAY_AGG(STRUCT(if(is_nullable = 'YES', true, false) AS required, column_name AS name, data_type AS type, if(column_default = 'NULL', null, column_default) AS \`default\`) ORDER BY ordinal_position) AS schema \
+FROM \`{dataset.id}\`.INFORMATION_SCHEMA.COLUMNS \
+GROUP BY table_name".replace('{dataset.id}', dataset.id)
+    const [rows] = await bq.query(query)
+    for (const row of rows) {
+			schema[dataset.id][row.table_name] = {}
+      for (const col of row.schema) {
+        const colName = col.name
+        delete col.name
+        if (col.default === null) {
+          delete col.default
+        }
+        schema[dataset.id][row.table_name][colName] = col
+      }
+    }
+  }
+  return schema
+}`, // nested template literals
+			lang: 'bun',
+			argName: 'args'
 		},
 		snowflake: {
-			code: `# requirements:
-# snowflake-connector-python==3.2.0
-from typing import Any
-import snowflake.connector as sf
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
-def main(args):
-    if not args["database"]:
-        raise Exception("a selected database is required for the schema explorer")
-    p_key = serialization.load_pem_private_key(
-        args["private_key"].encode(), password=None, backend=default_backend()
-    )
-    pkb = p_key.private_bytes(
-        encoding=serialization.Encoding.DER,
-        format=serialization.PrivateFormat.PKCS8,
-        encryption_algorithm=serialization.NoEncryption(),
-    )
-    ctx = sf.connect(
-        user=args["username"],
-        account=args["account_identifier"],
-        private_key=pkb,
-        warehouse=args["warehouse"],
-        database=args["database"],
-        schema=args["schema"],
-        role=args["role"],
-    )
-    cs = ctx.cursor()
-    rows = cs.execute("select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'")
-    schema = dict()
-    for row in rows:
-        if row[0] not in schema:
-            schema[row[0]] = dict()
-        if row[1] not in schema[row[0]]:
-            schema[row[0]][row[1]] = dict()
-        schema[row[0]][row[1]][row[3]] = {
-            "type": row[2],
-            "required": row[5] == "YES",
-        }
-        if row[4] is not None:
-            schema[row[0]][row[1]][row[3]]["default"] = row[4]
-    return schema
-`,
-			lang: 'python3'
+			code: `select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'`,
+			lang: 'snowflake',
+			processingFn: (rows) => {
+				const schema = {}
+				for (const row of rows) {
+					if (!(row.TABLE_SCHEMA in schema)) {
+						schema[row.TABLE_SCHEMA] = {}
+					}
+					if (!(row.TABLE_NAME in schema[row.TABLE_SCHEMA])) {
+						schema[row.TABLE_SCHEMA][row.TABLE_NAME] = {}
+					}
+					schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME] = {
+						type: row.DATA_TYPE,
+						required: row.IS_NULLABLE === 'YES'
+					}
+					if (row.COLUMN_DEFAULT !== null) {
+						schema[row.TABLE_SCHEMA][row.TABLE_NAME][row.COLUMN_NAME]['default'] =
+							row.COLUMN_DEFAULT
+					}
+				}
+				return schema
+			},
+			argName: 'database'
 		}
 	}
 
@@ -246,7 +179,7 @@ def main(args):
 				language: scripts[resourceType].lang as Preview.language,
 				content: scripts[resourceType].code,
 				args: {
-					args: '$res:' + resourcePath
+					[scripts[resourceType].argName]: '$res:' + resourcePath
 				}
 			}
 		})
@@ -263,10 +196,13 @@ def main(args):
 					} else {
 						if (resourceType !== undefined) {
 							if (resourceType !== 'graphql') {
+								const { processingFn } = scripts[resourceType]
+								const schema =
+									processingFn !== undefined ? processingFn(testResult.result) : testResult.result
 								$dbSchemas[resourcePath] = {
 									lang: resourceType as SQLSchema['lang'],
-									schema: testResult.result,
-									publicOnly: !!testResult.result.public || !!testResult.result.PUBLIC
+									schema,
+									publicOnly: !!schema.public || !!schema.PUBLIC
 								}
 							} else {
 								$dbSchemas[resourcePath] = {

frontend/src/lib/components/DisplayResult.svelte

@@ -14,6 +14,7 @@
 	export let disableExpand = false
 	export let jobId: string | undefined = undefined
 	export let workspaceId: string | undefined = undefined
+	export let disableDetails = false
 
 	let resultKind:
 		| 'json'
@@ -132,15 +133,18 @@
 				>
 				<slot name="copilot-fix" />
 			</div>
-		{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}<div
-				class="mb-2 w-full min-w-[400px] text-sm relative"
-				>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
-				{#if !disableExpand}
-					<div class="text-tertiary text-xs absolute top-5.5 right-0 inline-flex gap-2">
-						<button on:click={() => copyToClipboard(jsonStr)}><ClipboardCopy size={16} /></button>
-						<button on:click={jsonViewer.openDrawer}><Expand size={16} /></button>
-					</div>
-				{/if}
+		{/if}
+		{#if typeof result == 'object' && Object.keys(result).length > 0}
+			<div class="mb-2 w-full min-w-[400px] text-sm relative">
+			{#if !disableDetails}
+			The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
+			{/if}
+			{#if !disableExpand}
+				<div class="text-tertiary text-xs absolute top-5.5 right-0 inline-flex gap-2">
+					<button on:click={() => copyToClipboard(jsonStr)}><ClipboardCopy size={16} /></button>
+					<button on:click={jsonViewer.openDrawer}><Expand size={16} /></button>
+				</div>
+			{/if}
 			</div>{/if}{#if !forceJson && resultKind == 'table-col'}<div
 				class="grid grid-flow-col-dense border rounded-md"
 			>
@@ -236,7 +240,7 @@
 		{:else if !forceJson && resultKind == 'file'}
 			<div
 				><a
-					download={result.filename ?? 'windmill.file'}
+					download={result.filename ?? result.file?.filename ?? 'windmill.file'}
 					href="data:application/octet-stream;base64,{contentOrRootString(result.file)}">Download</a
 				>
 			</div>
@@ -296,6 +300,7 @@
 	{/if}
 </div>
 
+
 {#if !disableExpand}
 	<Portal>
 		<Drawer bind:this={jsonViewer} size="900px">

frontend/src/lib/components/Editor.svelte

@@ -24,6 +24,8 @@
 	import 'monaco-editor/esm/vs/basic-languages/graphql/graphql.contribution'
 	import 'monaco-editor/esm/vs/basic-languages/powershell/powershell.contribution'
 	import 'monaco-editor/esm/vs/language/typescript/monaco.contribution'
+	import 'monaco-editor/esm/vs/basic-languages/css/css.contribution'
+
 	import { MonacoLanguageClient, initServices } from 'monaco-languageclient'
 	import { toSocket, WebSocketMessageReader, WebSocketMessageWriter } from 'vscode-ws-jsonrpc'
 	import { CloseAction, ErrorAction, RequestType, NotificationType } from 'vscode-languageclient'
@@ -49,7 +51,15 @@
 	let divEl: HTMLDivElement | null = null
 	let editor: meditor.IStandaloneCodeEditor
 
-	export let lang: 'typescript' | 'python' | 'go' | 'shell' | 'sql' | 'graphql' | 'powershell'
+	export let lang:
+		| 'typescript'
+		| 'python'
+		| 'go'
+		| 'shell'
+		| 'sql'
+		| 'graphql'
+		| 'powershell'
+		| 'css'
 	export let deno: boolean
 	export let code: string = ''
 	export let cmdEnterAction: (() => void) | undefined = undefined

frontend/src/lib/components/FlowBuilder.svelte

@@ -421,7 +421,7 @@
 
 	let abortController: AbortController | undefined = undefined
 	let copilotLoading = false
-	let flowCopilotMode: 'trigger' | 'sequence' = 'trigger'
+	let flowCopilotMode: 'trigger' | 'sequence' = 'sequence'
 	let copilotStatus: string = ''
 	let copilotFlowInputs: Record<string, SchemaProperty> = {}
 	let copilotFlowRequiredInputs: string[] = []

frontend/src/lib/components/FlowPreviewContent.svelte

@@ -34,6 +34,9 @@
 		return modules
 			.filter((x) => idOrders.indexOf(x.id) <= upTo)
 			.map((m) => {
+				if (idOrders.indexOf(m.id) == upTo) {
+					return m
+				}
 				if (m.value.type === 'forloopflow') {
 					m.value.modules = sliceModules(m.value.modules, upTo, idOrders)
 				} else if (m.value.type === 'branchone') {

frontend/src/lib/components/HighlightCode.svelte

@@ -42,6 +42,7 @@
 				return sql
 			case 'powershell':
 				return powershell
+
 			default:
 				return typescript
 		}