fix(frontend): fix chartjs options

* feat: big query schema explorer

* fix: schema caching

.github/Dockerfile

@@ -1,7 +1,7 @@
 FROM nikolaik/python-nodejs
 
 RUN npm install -g @apidevtools/swagger-cli
-RUN pip install openapi-python-client
+RUN pip install openapi-python-client==0.15.1
 RUN pip install poetry
 
 

.github/DockerfilePypiBuilder

@@ -4,4 +4,4 @@ RUN python3 -m pip install pipx poetry
 RUN python3 -m pipx ensurepath
 ENV PATH="/root/.local/bin:${PATH}"
 ENV PATH="/usr/local/bin:${PATH}"
-RUN pipx install openapi-python-client==0.11.6 --include-deps
+RUN pipx install openapi-python-client==0.15.1 --include-deps

.github/workflows/benchmark.yml

@@ -0,0 +1,54 @@
+name: Run benchmarks
+
+on:
+  schedule:
+    - cron: "0 0 */1 * *"
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    runs-on: [self-hosted, new]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+      windmill:
+        image: ghcr.io/windmill-labs/windmill:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+    steps:
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+      - uses: actions/checkout@v4
+        with:
+          ref: benchmarks
+      - name: benchmark
+        timeout-minutes: 10
+        run: deno run --unstable -A
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          --host http://localhost:8000
+          -e admin@windmill.dev
+          -p changeme
+          -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
+          --branch ${GITHUB_REF##ref/head/}
+      - name: Push changes
+        run: |
+          pwd
+          git add .
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git commit -m "Update benchmarks"
+          git push

.github/workflows/docker-image.yml

@@ -96,6 +96,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
             features=enterprise

.github/workflows/docker-mssql.yml

@@ -3,7 +3,10 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 name: Build windmill:mssql
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 */4 * *"
 
 concurrency:
   group: ${{ github.ref }}-mssql

CHANGELOG.md

@@ -1,6 +1,145 @@
 # Changelog
 
 
+## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
+
+
+### Bug Fixes
+
+* add list resource types names ([fbbab5c](https://github.com/windmill-labs/windmill/commit/fbbab5c874748547a9ff3e58c1b7b22c90766f4f))
+* add stable ids to rows in AppTable ([0c91581](https://github.com/windmill-labs/windmill/commit/0c91581fcdf3a141f36e34610935aa100fcfee52))
+* reduce aggregate period to list users in workspace ([6bc0e37](https://github.com/windmill-labs/windmill/commit/6bc0e373fc6088636f09d217e8800a32337291ea))
+
+## [1.168.2](https://github.com/windmill-labs/windmill/compare/v1.168.1...v1.168.2) (2023-09-06)
+
+
+### Bug Fixes
+
+* fix sqlx build ([64e7fb5](https://github.com/windmill-labs/windmill/commit/64e7fb56e41b45bc2476d0e98fa99dcbc355cfe0))
+
+## [1.168.1](https://github.com/windmill-labs/windmill/compare/v1.168.0...v1.168.1) (2023-09-06)
+
+
+### Bug Fixes
+
+* fix sqlx build ([92c8146](https://github.com/windmill-labs/windmill/commit/92c8146a5778290b5a76c2ea5685f95b85be2e38))
+
+## [1.168.0](https://github.com/windmill-labs/windmill/compare/v1.167.0...v1.168.0) (2023-09-06)
+
+
+### Features
+
+* dedicated workers for native-throughput performance  (EE only)  ([#2239](https://github.com/windmill-labs/windmill/issues/2239)) ([c80f155](https://github.com/windmill-labs/windmill/commit/c80f155602eca972842be7bd560395a06e4e0ae6))
+
+
+### Bug Fixes
+
+* **frontend:** add virtual list ([#2218](https://github.com/windmill-labs/windmill/issues/2218)) ([e4c896b](https://github.com/windmill-labs/windmill/commit/e4c896b4b9f28b2fa219be249a2794faf3f1b7d0))
+
+## [1.167.1](https://github.com/windmill-labs/windmill/compare/v1.167.0...v1.167.1) (2023-09-05)
+
+
+### Bug Fixes
+
+* **frontend:** add virtual list ([#2218](https://github.com/windmill-labs/windmill/issues/2218)) ([e4c896b](https://github.com/windmill-labs/windmill/commit/e4c896b4b9f28b2fa219be249a2794faf3f1b7d0))
+
+## [1.167.0](https://github.com/windmill-labs/windmill/compare/v1.166.1...v1.167.0) (2023-09-04)
+
+
+### Features
+
+* submit result in background thread (unify architecture for dedicated worker) ([#2226](https://github.com/windmill-labs/windmill/issues/2226)) ([dff1cd9](https://github.com/windmill-labs/windmill/commit/dff1cd9a64f755f239eb57599c104c47f4d33b12))
+
+
+### Bug Fixes
+
+* **cli:** prioritize correctly content file to resolve for ts types ([2906d53](https://github.com/windmill-labs/windmill/commit/2906d535a126f4fe2cfe6dffda46e5fe841056da))
+
+## [1.166.1](https://github.com/windmill-labs/windmill/compare/v1.166.0...v1.166.1) (2023-09-03)
+
+
+### Bug Fixes
+
+* fix setting is ready for s3 workers ([b0ed0f9](https://github.com/windmill-labs/windmill/commit/b0ed0f964843247d11ecfe586f1565589df95ff6))
+
+## [1.166.0](https://github.com/windmill-labs/windmill/compare/v1.165.0...v1.166.0) (2023-09-03)
+
+
+### Features
+
+* **frontend:** App stepper debug ([#2202](https://github.com/windmill-labs/windmill/issues/2202)) ([77f8eac](https://github.com/windmill-labs/windmill/commit/77f8eac21e0edfa1eada617d78a498a3a6ae1dce))
+
+
+### Bug Fixes
+
+* fix datetime handling for python ([b35ffd4](https://github.com/windmill-labs/windmill/commit/b35ffd435de97ed34fcda69490abd734ea3229fa))
+* **frontend:** Fix App Modal z-index ([#2210](https://github.com/windmill-labs/windmill/issues/2210)) ([9787edb](https://github.com/windmill-labs/windmill/commit/9787edb67c329265bf179fe304d00cdc1df7042e))
+* see run detail in a new tab ([719a7b1](https://github.com/windmill-labs/windmill/commit/719a7b11da81f68452ba9fc22ff456fe1ddde1de))
+* update wmill python generator thus updating windmill-api ([f912f1d](https://github.com/windmill-labs/windmill/commit/f912f1de86e91c5cdbc0012e2362467c4965936a))
+
+
+### Performance Improvements
+
+* improve queue performance ([#2222](https://github.com/windmill-labs/windmill/issues/2222)) ([069e2d1](https://github.com/windmill-labs/windmill/commit/069e2d18d586aa3d407e3b089d1ad94b2b838af0))
+
+## [1.165.0](https://github.com/windmill-labs/windmill/compare/v1.164.0...v1.165.0) (2023-08-31)
+
+
+### Features
+
+* improve queue performance when queue grows large ([ada88a2](https://github.com/windmill-labs/windmill/commit/ada88a2bf94fec71187bbdb210065de43d4cd3fb))
+* support partial go dependency pinning ([41107c7](https://github.com/windmill-labs/windmill/commit/41107c7cfa7b56099a9c8b08cfb16ff3cf840ff2))
+
+
+### Bug Fixes
+
+* uniformize that all job links specify the workspace ([d311d76](https://github.com/windmill-labs/windmill/commit/d311d76557432a72a5d6d7ab010aeb1fe0e599de))
+
+## [1.164.0](https://github.com/windmill-labs/windmill/compare/v1.163.1...v1.164.0) (2023-08-31)
+
+
+### Features
+
+* add workspace variable to worker tag ([276cd6d](https://github.com/windmill-labs/windmill/commit/276cd6dac39b7cb181ac46e3edea79a3a3bcff8d))
+
+
+### Bug Fixes
+
+* **frontend:** allow using Docker in Flow ([#2201](https://github.com/windmill-labs/windmill/issues/2201)) ([bb749c1](https://github.com/windmill-labs/windmill/commit/bb749c14f877f7cb1e8642b881a00aedfeb08f7d))
+
+## [1.163.1](https://github.com/windmill-labs/windmill/compare/v1.163.0...v1.163.1) (2023-08-30)
+
+
+### Bug Fixes
+
+* avoid perpetual spinning of recompute all component ([11e1ecb](https://github.com/windmill-labs/windmill/commit/11e1ecbcda92f5ab643b776094ef10005d51b579))
+
+## [1.163.0](https://github.com/windmill-labs/windmill/compare/v1.162.2...v1.163.0) (2023-08-30)
+
+
+### Features
+
+* add global cache configuration ([7c5ea56](https://github.com/windmill-labs/windmill/commit/7c5ea569a8102ef052d42216e2ff8d4c3169a7a5))
+
+
+### Bug Fixes
+
+* fix cyclical loop in apps ([61df339](https://github.com/windmill-labs/windmill/commit/61df339343767e63cbe7a4e75f1fd4f848dbd7e0))
+
+## [1.162.2](https://github.com/windmill-labs/windmill/compare/v1.162.1...v1.162.2) (2023-08-29)
+
+
+### Bug Fixes
+
+* fix incorrect bump ([4704899](https://github.com/windmill-labs/windmill/commit/4704899a81cb281b99949c934184e23b199b2ed8))
+
+## [1.162.1](https://github.com/windmill-labs/windmill/compare/v1.162.0...v1.162.1) (2023-08-29)
+
+
+### Bug Fixes
+
+* fix deps incompatibilities ([6c5a8a3](https://github.com/windmill-labs/windmill/commit/6c5a8a3613b4608e6d2b57e7f40cd4ab2d1af9ae))
+
 ## [1.162.0](https://github.com/windmill-labs/windmill/compare/v1.161.0...v1.162.0) (2023-08-29)
 
 

Dockerfile

@@ -96,8 +96,8 @@ SHELL ["/bin/bash", "-c"]
 RUN apt update -y
 RUN apt install -y unzip curl
 
-RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.2/deno-linux-arm64.zip -o deno.zip || true
-RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.2/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.3/deno-linux-arm64.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.3/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
 
 RUN unzip deno.zip && rm deno.zip
 
@@ -164,6 +164,9 @@ RUN set -eux; \
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
+# go build is slower the first time it is ran, so we prewarm it in the build
+RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm &&  printf "package foo\nimport (\"fmt\")\nfunc main() { fmt.Println(42) }" > warm.go && go build -x && rm -rf /tmp/gobuildwarm
+
 ENV TZ=Etc/UTC
 
 RUN /usr/local/bin/python3 -m pip install pip-tools

README.md

@@ -85,7 +85,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
    shared on [WindmillHub](https://hub.windmill.dev).
    ![Step 3](./imgs/windmill-flow.png)
 
-4. Build complex UI on top of your scripts and flows.
+4. Build complex UIs on top of your scripts and flows.
    ![Step 4](./imgs/windmill-builder.png)
 
 Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or

backend/.gitignore

@@ -1,4 +1,5 @@
 target/
 .env
 oauth.json
-windmill-api/openapi-deref.yaml
+windmill-api/openapi-deref.yaml
+tracing.folded

backend/.sqlx/query-0694f4be1a7966126f20afbd9ee094088db8ce98aefdbe96144c42e38e5bd813.json

@@ -1,41 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "tag",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "concurrent_limit",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 2,
-        "name": "concurrency_time_window_s",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 3,
-        "name": "cache_ttl",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "0694f4be1a7966126f20afbd9ee094088db8ce98aefdbe96144c42e38e5bd813"
-}

backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json

@@ -0,0 +1,74 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where hash = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "concurrent_limit",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 2,
+        "name": "concurrency_time_window_s",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "cache_ttl",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd"
+}

backend/.sqlx/query-15ef5759a2ccd7b7f9fd3f2ce0d54d01fe0a2c7e9692ac4ce29a86eb509e1a1d.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token\n                    (token, label, super_admin, email)\n                    VALUES ($1, $2, $3, $4)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "15ef5759a2ccd7b7f9fd3f2ce0d54d01fe0a2c7e9692ac4ce29a86eb509e1a1d"
+}

backend/.sqlx/query-1c1577b9963d907c4245a027fece57285ce64ac41a84681b32deb71e452334c1.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM global_settings WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "1c1577b9963d907c4245a027fece57285ce64ac41a84681b32deb71e452334c1"
+}

backend/.sqlx/query-1e232d4b978d133ba542d0abe5afaa0031f630c5336c0b805590b5ea4508ed04.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "VACUUM queue",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "1e232d4b978d133ba542d0abe5afaa0031f630c5336c0b805590b5ea4508ed04"
+}

backend/.sqlx/query-330e85c6fe52355971262d3a44c2b75a95b1d042b9e0995186aa5a2d0b7ee552.json

@@ -0,0 +1,30 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) AS completed_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND job_kind != 'dependencies' AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at\n                FROM queue\n                WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "min_started_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 1,
+        "name": "completed_count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Float8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true
+    ]
+  },
+  "hash": "330e85c6fe52355971262d3a44c2b75a95b1d042b9e0995186aa5a2d0b7ee552"
+}

backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -58,10 +58,11 @@
         "VarcharArray",
         "Int4",
         "Int4",
-        "Int4"
+        "Int4",
+        "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "52dbe77fad5729e9e121609eeb6a3573e6d0e1d1c16e29b15c7ad810536e249c"
+  "hash": "438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc"
 }

backend/.sqlx/query-45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT * FROM resource WHERE workspace_id = $1",
+  "query": "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
   "describe": {
     "columns": [
       {
@@ -48,5 +48,5 @@
       false
     ]
   },
-  "hash": "fa258894bd90ea6586669e5810c5c6bcb42d5e1e68fab27fb185d06962b6454a"
+  "hash": "45e4d13f5806122faecdb1d9ab18159555b652869a036b006f4a151e999b17b7"
 }

backend/.sqlx/query-467247beca4dae321ab3acc3a99f655412761157cda54bbd3e58cada3de3396c.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "467247beca4dae321ab3acc3a99f655412761157cda54bbd3e58cada3de3396c"
+}

backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json

@@ -0,0 +1,80 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND archived = false)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "concurrent_limit",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "concurrency_time_window_s",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 5,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825"
+}

backend/.sqlx/query-637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875.json

@@ -1,44 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "language: ScriptLang",
-        "type_info": {
-          "Custom": {
-            "name": "script_lang",
-            "kind": {
-              "Enum": [
-                "python3",
-                "deno",
-                "go",
-                "bash",
-                "postgresql",
-                "nativets",
-                "Nativets",
-                "bun",
-                "mysql",
-                "bigquery",
-                "snowflake",
-                "graphql",
-                "powershell"
-              ]
-            }
-          }
-        }
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875"
-}

backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json

@@ -0,0 +1,80 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where hash = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "concurrent_limit",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "concurrency_time_window_s",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 5,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f"
+}

backend/.sqlx/query-71879f3644dde27f2d93e5511e420579dfd2ffab72bb8480a1622b68703d4ea7.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1 RETURNING counter",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "counter",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "71879f3644dde27f2d93e5511e420579dfd2ffab72bb8480a1622b68703d4ea7"
+}

backend/.sqlx/query-8062bb1ac475c4a2a016d9dfa63c95765356fe7744b817ad0c320a54a33b24f3.json

@@ -1,47 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "hash",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "tag",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "concurrent_limit",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 3,
-        "name": "concurrency_time_window_s",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 4,
-        "name": "cache_ttl",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "8062bb1ac475c4a2a016d9dfa63c95765356fe7744b817ad0c320a54a33b24f3"
-}

backend/.sqlx/query-86652246b20f9bd023d3419ec382176780a87f32673b53de6f899564c1494147.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO concurrency_counter VALUES ($1, 1)\n        ON CONFLICT (concurrency_id) \n        DO UPDATE SET counter = concurrency_counter.counter + 1\n        RETURNING concurrency_counter.counter",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "counter",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "86652246b20f9bd023d3419ec382176780a87f32673b53de6f899564c1494147"
+}

backend/.sqlx/query-a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a59b70164dc87224d09a04d5469ca217eb19a15a250c3b83ca63f606f89b9681"
+}

backend/.sqlx/query-afbae51dae93e19045aee8d2d9ea1d31609aa7947a52545dce4455a4983493a7.json

@@ -1,30 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND job_kind != 'dependencies' AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n                FROM queue\n                WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $3 AND canceled = false\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "min_started_at",
-        "type_info": "Timestamptz"
-      },
-      {
-        "ordinal": 1,
-        "name": "total_count",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Float8",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      true
-    ]
-  },
-  "hash": "afbae51dae93e19045aee8d2d9ea1d31609aa7947a52545dce4455a4983493a7"
-}

backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json

@@ -0,0 +1,80 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "concurrent_limit",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "concurrency_time_window_s",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "cache_ttl",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 5,
+        "name": "language: ScriptLang",
+        "type_info": {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "Nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192"
+}

backend/.sqlx/query-cec906e3bcfe5b477f23e8bb44500b97166c789ab1cf6487679d092eebf3a48b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT COUNT(*) FROM group_ WHERE name != 'all'",
+  "query": "SELECT COUNT(*) FROM group_ WHERE name != 'all' AND name != 'error_handler' AND name != 'slack'",
   "describe": {
     "columns": [
       {
@@ -16,5 +16,5 @@
       null
     ]
   },
-  "hash": "18bdd06f4791c652addb4783f419f5d020c1d23278f583c6be6314768f9bf25a"
+  "hash": "cec906e3bcfe5b477f23e8bb44500b97166c789ab1cf6487679d092eebf3a48b"
 }

backend/.sqlx/query-cf21b61aa5b5e47744235c860e2d9b90d7654a4b5d0ec32085b178399f701bfc.json

@@ -1,47 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND archived = false)",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "hash",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "tag",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "concurrent_limit",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 3,
-        "name": "concurrency_time_window_s",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 4,
-        "name": "cache_ttl",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "cf21b61aa5b5e47744235c860e2d9b90d7654a4b5d0ec32085b178399f701bfc"
-}

backend/.sqlx/query-ebac391c83dae67831502fbe3f1034210afce7902a69f7c42bb68160e0464d71.json

@@ -1,47 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "path",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 1,
-        "name": "tag",
-        "type_info": "Varchar"
-      },
-      {
-        "ordinal": 2,
-        "name": "concurrent_limit",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 3,
-        "name": "concurrency_time_window_s",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 4,
-        "name": "cache_ttl",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "ebac391c83dae67831502fbe3f1034210afce7902a69f7c42bb68160e0464d71"
-}

backend/.sqlx/query-ed5764ead5e8f433adfe666976664876b22dcb2b34dcd99f1e44cd03cb9ab4ef.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      false
+    ]
+  },
+  "hash": "ed5764ead5e8f433adfe666976664876b22dcb2b34dcd99f1e44cd03cb9ab4ef"
+}

backend/Cargo.lock

@@ -64,9 +64,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
+checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
 dependencies = [
  "memchr",
 ]
@@ -163,9 +163,9 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
 [[package]]
 name = "argon2"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2e554a8638bdc1e4eae9984845306cc95f8a9208ba8d49c3859fd958b46774d"
+checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9"
 dependencies = [
  "base64ct",
  "blake2",
@@ -204,7 +204,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -328,13 +328,13 @@ dependencies = [
 
 [[package]]
 name = "async-recursion"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
+checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -411,7 +411,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -604,21 +604,21 @@ dependencies = [
  "lazycell",
  "log",
  "peeking_take_while",
- "prettyplease 0.2.12",
+ "prettyplease 0.2.14",
  "proc-macro2",
  "quote",
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.29",
+ "syn 2.0.31",
  "which",
 ]
 
 [[package]]
 name = "bindgen"
-version = "0.66.1"
+version = "0.68.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7"
+checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
 dependencies = [
  "bitflags 2.4.0",
  "cexpr",
@@ -631,7 +631,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -922,9 +922,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.27"
+version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f56b4c72906975ca04becb8a30e102dfecddd0c06181e3e95ddc444be28881f8"
+checksum = "d87d9d13be47a5b7c3907137f1290b0459a7f80efb26be8c52afb11963bccb02"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -980,20 +980,19 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.4.1"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27"
+checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6"
 dependencies = [
  "clap_builder",
  "clap_derive",
- "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.4.1"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d"
+checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1003,14 +1002,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.4.0"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a"
+checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1317,7 +1316,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1339,14 +1338,14 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
 name = "dashmap"
-version = "5.5.2"
+version = "5.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b101bb8960ab42ada6ae98eb82afcea4452294294c45b681295af26610d6d28"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
  "hashbrown 0.14.0",
@@ -1397,7 +1396,7 @@ checksum = "3c65c2ffdafc1564565200967edc4851c7b55422d3913466688907efd05ea26f"
 dependencies = [
  "deno-proc-macro-rules-macros",
  "proc-macro2",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1409,7 +1408,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1529,7 +1528,7 @@ dependencies = [
  "strum",
  "strum_macros",
  "syn 1.0.109",
- "syn 2.0.29",
+ "syn 2.0.31",
  "thiserror",
 ]
 
@@ -1864,6 +1863,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "finl_unicode"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6"
+
 [[package]]
 name = "fixedbitset"
 version = "0.4.2"
@@ -1933,7 +1938,7 @@ dependencies = [
  "pmutil",
  "proc-macro2",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1961,7 +1966,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e"
 dependencies = [
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1973,7 +1978,7 @@ dependencies = [
  "frunk_core",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1985,7 +1990,7 @@ dependencies = [
  "frunk_core",
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2086,7 +2091,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2326,12 +2331,11 @@ dependencies = [
 
 [[package]]
 name = "headers"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584"
+checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
 dependencies = [
- "base64 0.13.1",
- "bitflags 1.3.2",
+ "base64 0.21.3",
  "bytes",
  "headers-core",
  "http",
@@ -2563,6 +2567,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
  "hashbrown 0.12.3",
+ "serde",
 ]
 
 [[package]]
@@ -2573,7 +2578,6 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.0",
- "serde",
 ]
 
 [[package]]
@@ -2618,7 +2622,7 @@ dependencies = [
  "pmutil",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2628,7 +2632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi",
- "rustix 0.38.10",
+ "rustix 0.38.11",
  "windows-sys",
 ]
 
@@ -3077,9 +3081,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.6.0"
+version = "2.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c"
+checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
 
 [[package]]
 name = "memoffset"
@@ -3146,7 +3150,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "termcolor",
  "thiserror",
 ]
@@ -3192,7 +3196,7 @@ checksum = "57349d5a326b437989b6ee4dc8f2f34b0cc131202748414712a8e7d98952fc8c"
 dependencies = [
  "base64 0.21.3",
  "bigdecimal",
- "bindgen 0.66.1",
+ "bindgen 0.68.1",
  "bitflags 2.4.0",
  "bitvec",
  "byteorder",
@@ -3349,9 +3353,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.32.0"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "memchr",
 ]
@@ -3370,11 +3374,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openapiv3"
-version = "1.0.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75e56d5c441965b6425165b7e3223cc933ca469834f4a8b4786817a1f9dc4f13"
+checksum = "7b1a9f106eb0a780abd17ba9fca8e0843e3461630bcbe2af0ad4d5d3ba4e9aa4"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap 1.9.3",
  "serde",
  "serde_json",
 ]
@@ -3402,7 +3406,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3413,9 +3417,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.92"
+version = "0.9.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b"
+checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
 dependencies = [
  "cc",
  "libc",
@@ -3557,10 +3561,11 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "pest"
-version = "2.7.2"
+version = "2.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a"
+checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33"
 dependencies = [
+ "memchr",
  "thiserror",
  "ucd-trie",
 ]
@@ -3660,7 +3665,7 @@ dependencies = [
  "phf_shared 0.11.2",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3698,7 +3703,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3770,7 +3775,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3874,12 +3879,12 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.12"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
+checksum = "8832c0f9be7e3cae60727e6256cfd2cd3c3e2b6cd5dad4190ecb2fd658c9030b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3999,7 +4004,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "syn 2.0.29",
+ "syn 2.0.31",
  "thiserror",
  "typify",
  "unicode-ident",
@@ -4019,7 +4024,7 @@ dependencies = [
  "serde_json",
  "serde_tokenstream",
  "serde_yaml",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -4175,9 +4180,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.23.2"
+version = "0.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
+checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
 dependencies = [
  "async-std",
  "async-trait",
@@ -4226,13 +4231,13 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.4"
+version = "1.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
+checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.3.7",
+ "regex-automata 0.3.8",
  "regex-syntax 0.7.5",
 ]
 
@@ -4247,9 +4252,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
+checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -4466,7 +4471,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.29",
+ "syn 2.0.31",
  "walkdir",
 ]
 
@@ -4556,9 +4561,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.10"
+version = "0.38.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964"
+checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453"
 dependencies = [
  "bitflags 2.4.0",
  "errno",
@@ -4904,7 +4909,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -4980,7 +4985,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5087,9 +5092,9 @@ dependencies = [
 
 [[package]]
 name = "shlex"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
 
 [[package]]
 name = "signal-hook-registry"
@@ -5279,11 +5284,11 @@ dependencies = [
 
 [[package]]
 name = "sqlformat"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e"
+checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85"
 dependencies = [
- "itertools 0.10.5",
+ "itertools 0.11.0",
  "nom",
  "unicode_categories",
 ]
@@ -5555,15 +5560,16 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
 name = "stringprep"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da"
+checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6"
 dependencies = [
+ "finl_unicode",
  "unicode-bidi",
  "unicode-normalization",
 ]
@@ -5593,7 +5599,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5675,7 +5681,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5724,7 +5730,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5807,7 +5813,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5911,7 +5917,7 @@ dependencies = [
  "pmutil",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5923,7 +5929,7 @@ dependencies = [
  "pmutil",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5947,7 +5953,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "swc_macros_common",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5963,9 +5969,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.29"
+version = "2.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5993,7 +5999,7 @@ dependencies = [
  "cfg-if",
  "fastrand 2.0.0",
  "redox_syscall 0.3.5",
- "rustix 0.38.10",
+ "rustix 0.38.11",
  "windows-sys",
 ]
 
@@ -6028,22 +6034,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.47"
+version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
+checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.47"
+version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
+checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -6158,7 +6164,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -6361,9 +6367,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82"
+checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
 dependencies = [
  "bitflags 2.4.0",
  "bytes",
@@ -6411,7 +6417,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -6424,6 +6430,17 @@ dependencies = [
  "valuable",
 ]
 
+[[package]]
+name = "tracing-flame"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bae117ee14789185e129aaee5d93750abe67fdc5a9a62650452bfe4e122a3a9"
+dependencies = [
+ "lazy_static",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-futures"
 version = "0.2.5"
@@ -6538,7 +6555,7 @@ dependencies = [
  "regress",
  "schemars",
  "serde_json",
- "syn 2.0.29",
+ "syn 2.0.31",
  "thiserror",
  "unicode-ident",
 ]
@@ -6555,7 +6572,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.29",
+ "syn 2.0.31",
  "typify-impl",
 ]
 
@@ -6823,9 +6840,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
 
 [[package]]
 name = "walkdir"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
+checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
 dependencies = [
  "same-file",
  "winapi-util",
@@ -6879,7 +6896,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "wasm-bindgen-shared",
 ]
 
@@ -6913,7 +6930,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -6973,9 +6990,9 @@ dependencies = [
 
 [[package]]
 name = "webpki"
-version = "0.22.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
+checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e"
 dependencies = [
  "ring",
  "untrusted",
@@ -7016,13 +7033,14 @@ checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
 
 [[package]]
 name = "which"
-version = "4.4.0"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
 dependencies = [
  "either",
- "libc",
+ "home",
  "once_cell",
+ "rustix 0.38.11",
 ]
 
 [[package]]
@@ -7068,7 +7086,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -7103,7 +7121,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "argon2",
@@ -7162,10 +7180,11 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "base64 0.21.3",
  "chrono",
+ "openapiv3",
  "prettyplease 0.1.25",
  "progenitor",
  "progenitor-client",
@@ -7179,7 +7198,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "chrono",
  "serde",
@@ -7192,7 +7211,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -7211,13 +7230,14 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "tracing-flame",
  "tracing-subscriber",
  "uuid 1.4.1",
 ]
 
 [[package]]
 name = "windmill-parser"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "serde",
  "serde_json",
@@ -7225,7 +7245,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -7236,17 +7256,19 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "gosyn",
  "itertools 0.11.0",
+ "lazy_static",
+ "regex",
  "windmill-parser",
 ]
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -7257,7 +7279,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "itertools 0.11.0",
@@ -7268,7 +7290,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -7285,7 +7307,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -7296,7 +7318,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -7313,7 +7335,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "getrandom 0.2.10",
@@ -7331,7 +7353,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -7360,7 +7382,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.162.0"
+version = "1.168.3"
 dependencies = [
  "anyhow",
  "async-recursion",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.162.0"
+version = "1.168.3"
 authors.workspace = true
 edition.workspace = true
 
@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.162.0"
+version = "1.168.3"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -36,7 +36,8 @@ incremental = true
 
 [features]
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
-benchmark = ["windmill-api/benchmark"]
+benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
+flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 
 [dependencies]
 anyhow.workspace = true
@@ -188,4 +189,4 @@ samael = { version = "0.0.12", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
 rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
-pem = "3.0.1"
+pem = "3.0.1"

backend/migrations/20230829202651_add_global_settings.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230829202651_add_global_settings.up.sql

@@ -0,0 +1,6 @@
+-- Add up migration script here
+CREATE TABLE global_settings (
+    name VARCHAR(255) PRIMARY KEY,
+    value JSONB NOT NULL,
+    updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+);

backend/migrations/20230831210028_add_q_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230831210028_add_q_index.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS queue_sort ON queue (scheduled_for, created_at) WHERE running = false;

backend/migrations/20230902192640_add_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230902192640_add_index.up.sql

@@ -0,0 +1,13 @@
+-- Add up migration script here
+DROP INDEX IF EXISTS queue_sort;
+CREATE INDEX IF NOT EXISTS queue_sort ON queue (scheduled_for, created_at, tag) WHERE running = false;
+
+CREATE TABLE IF NOT EXISTS concurrency_counter (
+  concurrency_id VARCHAR(1000) PRIMARY KEY,
+  counter INTEGER NOT NULL
+);
+
+DROP INDEX IF EXISTS root_queue_index_suspended;
+
+DROP INDEX IF EXISTS queue_suspended;
+CREATE INDEX IF NOT EXISTS queue_suspended ON queue (created_at, suspend_until, suspend, tag) WHERE suspend_until is not null;

backend/migrations/20230905170218_dedicated_worker.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230905170218_dedicated_worker.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN dedicated_worker boolean;

backend/out.json

@@ -0,0 +1 @@
+{"completed":true,"result":{"error":{"name":"Exception","stack":"  File \"/tmp/u/ruben/gentle_script.py\", line 35, in main\n    print(wmill.get_variable(\"u/admin/adorable_postgresql\"))\n          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n  File \"/tmp/windmill/cache/pip/wmill==1.166.0/wmill/client.py\", line 336, in get_variable\n    raise Exception(\n","message":"Variable at path u/admin/adorable_postgresql does not exist or you do not have read permissions on it"}}}

backend/parsers/windmill-parser-go/Cargo.toml

@@ -12,4 +12,6 @@ path = "./src/lib.rs"
 windmill-parser.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
-gosyn.workspace = true
+gosyn.workspace = true
+lazy_static.workspace = true
+regex.workspace = true

backend/parsers/windmill-parser-go/src/lib.rs

@@ -6,8 +6,13 @@ use gosyn::{
 };
 use itertools::Itertools;
 
+use regex::Regex;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
 
+lazy_static::lazy_static! {
+    pub static ref REQUIRE_PARSE: Regex = Regex::new(r"//require (.*)\n").unwrap();
+}
+
 pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let filtered_code = filter_non_main(code);
     let file = parse_source(&filtered_code).map_err(|x| anyhow::anyhow!(x.to_string()))?;
@@ -46,7 +51,14 @@ pub fn parse_go_imports(code: &str) -> anyhow::Result<Vec<String>> {
         })
         .collect();
     imports.sort();
-    Ok(imports)
+    Ok([
+        imports,
+        REQUIRE_PARSE
+            .captures_iter(code)
+            .map(|x| x[1].to_string())
+            .collect_vec(),
+    ]
+    .concat())
 }
 
 fn get_name(param: &Field) -> String {

backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql

@@ -791,6 +791,7 @@ CREATE TABLE public.script (
     envs character varying(1000)[],
     concurrent_limit integer,
     concurrency_time_window_s integer,
+    dedicated_worker boolean,
     CONSTRAINT proper_id CHECK (((path)::text ~ '^[ufg](\/[\w-]+){2,}$'::text))
 );
 

backend/src/main.rs

@@ -20,7 +20,7 @@ use tokio::{
     sync::RwLock,
 };
 use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
-use windmill_common::{utils::rd_string, METRICS_ADDR};
+use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
 use windmill_worker::{
     BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
@@ -40,15 +40,19 @@ mod monitor;
 async fn main() -> anyhow::Result<()> {
     dotenv::dotenv().ok();
 
+    #[cfg(not(feature = "flamegraph"))]
     windmill_common::tracing_init::initialize_tracing();
 
+    #[cfg(feature = "flamegraph")]
+    let _guard = windmill_common::tracing_init::setup_flamegraph();
+
     let num_workers = std::env::var("NUM_WORKERS")
         .ok()
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(DEFAULT_NUM_WORKERS as i32);
 
     if num_workers > 1 {
-        tracing::warn!("We recommend using at most 1 worker per container, use more only if you know what you are doing.");
+        tracing::warn!("We STRONGLY recommend using at most 1 worker per container, unless this worker is dedicated to native jobs only. ");
     }
     let metrics_addr: Option<SocketAddr> = *METRICS_ADDR;
 
@@ -92,6 +96,7 @@ async fn main() -> anyhow::Result<()> {
     tracing::info!("Database connected");
 
     let rsmq = if let Some(config) = rsmq_config {
+        tracing::info!("Redis config set: {:?}", config);
         Some(rsmq_async::MultiplexedRsmq::new(config).await.unwrap())
     } else {
         None
@@ -119,68 +124,7 @@ Windmill Community Edition {GIT_VERSION}
 ##############################"
     );
 
-    display_config(vec![
-        "DISABLE_NSJAIL",
-        "DISABLE_SERVER",
-        "NUM_WORKERS",
-        "METRICS_ADDR",
-        "JSON_FMT",
-        "BASE_URL",
-        "TIMEOUT",
-        "ZOMBIE_JOB_TIMEOUT",
-        "RESTART_ZOMBIE_JOBS",
-        "SLEEP_QUEUE",
-        "MAX_LOG_SIZE",
-        "SERVER_BIND_ADDR",
-        "PORT",
-        "KEEP_JOB_DIR",
-        "S3_CACHE_BUCKET",
-        "TAR_CACHE_RATE",
-        "COOKIE_DOMAIN",
-        "PYTHON_PATH",
-        "DENO_PATH",
-        "GO_PATH",
-        "GOPRIVATE",
-        "GOPROXY",
-        "NETRC",
-        "PIP_INDEX_URL",
-        "PIP_EXTRA_INDEX_URL",
-        "PIP_TRUSTED_HOST",
-        "PATH",
-        "HOME",
-        "DATABASE_CONNECTIONS",
-        "TIMEOUT_WAIT_RESULT",
-        "QUEUE_LIMIT_WAIT_RESULT",
-        "DENO_AUTH_TOKENS",
-        "DENO_FLAGS",
-        "NPM_CONFIG_REGISTRY",
-        "PIP_LOCAL_DEPENDENCIES",
-        "ADDITIONAL_PYTHON_PATHS",
-        "INCLUDE_HEADERS",
-        "INSTANCE_EVENTS_WEBHOOK",
-        "CLOUD_HOSTED",
-        "GLOBAL_CACHE_INTERVAL",
-        "WORKER_TAGS",
-        "CUSTOM_TAGS",
-        "JOB_RETENTION_SECS",
-        "WAIT_RESULT_FAST_POLL_DURATION_SECS",
-        "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
-        "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
-        "EXIT_AFTER_NO_JOB_FOR_SECS",
-        "REQUEST_SIZE_LIMIT",
-        "SMTP_HOST",
-        "SMTP_USERNAME",
-        "SMTP_PORT",
-        "SMTP_TLS_IMPLICIT",
-        "CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
-        "GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
-    ]);
-
-    if std::env::var("WHITELIST_WORKSPACES").is_ok()
-        || std::env::var("BLACKLIST_WORKSPACES").is_ok()
-    {
-        panic!("WHITELIST_WORKSPACES and BLACKLIST_WORKSPACES have been removed, please use Worker Groups instead");
-    }
+    display_config(&ENV_SETTINGS);
 
     tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
     if let Some(ref smtp) = *SMTP_CLIENT {
@@ -264,7 +208,7 @@ Windmill Community Edition {GIT_VERSION}
     Ok(())
 }
 
-fn display_config(envs: Vec<&str>) {
+fn display_config(envs: &[&str]) {
     tracing::info!(
         "config: {}",
         envs.iter()

backend/src/monitor.rs

@@ -133,7 +133,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         let _ = handle_job_error(
             db,
             &client,
-            job,
+            &job,
             error::Error::ExecutionErr(format!(
                 "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
                 last_ping
@@ -145,7 +145,6 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             true,
             same_worker_tx_never_used,
             "",
-            base_internal_url,
             rsmq.clone(),
         )
         .await;

backend/windmill-api-client/Cargo.toml

@@ -19,9 +19,11 @@ uuid.workspace = true
 serde_json.workspace = true
 rand.workspace = true
 base64.workspace = true
+openapiv3 = "=1.0.2"
 
 [build-dependencies]
 prettyplease = "0.1.25"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 serde_json = "1.0"
-syn = "1.0"
+syn = "1.0"
+openapiv3 = "=1.0.2"

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.162.0
+  version: 1.168.3
   title: Windmill API
 
   contact:
@@ -522,6 +522,59 @@ paths:
               schema:
                 type: boolean
 
+  /settings/global/{key}:
+    get:
+      summary: get global settings
+      operationId: getGlobal
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/Key"
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
+    post:
+      summary: post global settings
+      operationId: setGlobal
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/Key"
+      requestBody:
+        description: value set
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                value: {}
+
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /settings/local:
+    get:
+      summary: get local settings
+      operationId: getLocal
+      tags:
+        - setting
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -1951,6 +2004,33 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableResource"
 
+  /w/{workspace}/resources/list_names/{name}:
+    get:
+      summary: list resource names
+      operationId: listResourceNames
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Name"
+      responses:
+        "200":
+          description: resource list names
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    path:
+                      type: string
+                  required:
+                    - name
+                    - path
+
   /w/{workspace}/resources/type/create:
     post:
       summary: create resource_type
@@ -5450,6 +5530,12 @@ components:
       name: token
 
   parameters:
+    Key:
+      name: key
+      in: path
+      required: true
+      schema:
+        type: string
     WorkspaceId:
       name: workspace
       in: path
@@ -5835,6 +5921,8 @@ components:
             type: integer
         cache_ttl:
           type: number
+        dedicated_worker:
+          type: boolean
       required:
         - hash
         - path
@@ -5910,6 +5998,8 @@ components:
             type: integer
         cache_ttl:
           type: number
+        dedicated_worker:
+          type: boolean
       required:
         - path
         - summary

backend/windmill-api/src/groups.rs

@@ -201,12 +201,12 @@ pub async fn require_is_owner(
 }
 
 async fn _check_nb_of_groups(db: &DB) -> Result<()> {
-    let nb_groups = sqlx::query_scalar!("SELECT COUNT(*) FROM group_ WHERE name != 'all'",)
+    let nb_groups = sqlx::query_scalar!("SELECT COUNT(*) FROM group_ WHERE name != 'all' AND name != 'error_handler' AND name != 'slack'",)
         .fetch_one(db)
         .await?;
     if nb_groups.unwrap_or(0) >= 5 {
         return Err(Error::BadRequest(
-            "You have reached the maximum number of groups without an enterprise license"
+            "You have reached the maximum number of groups (5 outside of native groups 'all', 'slack' and 'error_handler') without an enterprise license"
                 .to_string(),
         ));
     }
@@ -536,8 +536,7 @@ struct IGroup {
     name: String,
     emails: Option<Vec<String>>,
 }
-async fn list_igroups(authed: ApiAuthed, Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
-    require_super_admin(&db, &authed.email).await?;
+async fn list_igroups(Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
     let mut tx: Transaction<'_, Postgres> = db.begin().await?;
 
     let groups = sqlx::query_as!(

backend/windmill-api/src/jobs.rs

@@ -315,9 +315,11 @@ pub async fn get_path_tag_limits_cache_for_hash(
     Option<i32>,
     Option<i32>,
     Option<i32>,
+    ScriptLang,
+    Option<bool>,
 )> {
     let script = sqlx::query!(
-        "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
+        "select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where hash = $1 AND workspace_id = $2",
         hash,
         w_id
     )
@@ -334,6 +336,8 @@ pub async fn get_path_tag_limits_cache_for_hash(
         script.concurrent_limit,
         script.concurrency_time_window_s,
         script.cache_ttl,
+        script.language,
+        script.dedicated_worker,
     ))
 }
 
@@ -2035,8 +2039,15 @@ pub async fn run_wait_result_script_by_hash(
     check_queue_too_long(&db, run_query.queue_limit).await?;
 
     let hash = script_hash.0;
-    let (path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-        get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
+    let (
+        path,
+        tag,
+        concurrent_limit,
+        concurrency_time_window_s,
+        cache_ttl,
+        language,
+        dedicated_worker,
+    ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
     check_scopes(&authed, || format!("run:script/{path}"))?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
@@ -2054,6 +2065,8 @@ pub async fn run_wait_result_script_by_hash(
             concurrent_limit: concurrent_limit,
             concurrency_time_window_s: concurrency_time_window_s,
             cache_ttl,
+            language,
+            dedicated_worker,
         },
         args,
         &authed.username,
@@ -2253,6 +2266,7 @@ async fn run_preview_job(
     Ok((StatusCode::CREATED, uuid.to_string()))
 }
 
+#[tracing::instrument(level = "trace", skip_all)]
 async fn add_noop_jobs(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -2360,8 +2374,15 @@ pub async fn run_job_by_hash(
     JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
-    let (path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-        get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
+    let (
+        path,
+        tag,
+        concurrent_limit,
+        concurrency_time_window_s,
+        cache_ttl,
+        language,
+        dedicated_worker,
+    ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
     check_scopes(&authed, || format!("run:script/{path}"))?;
 
     let scheduled_for = run_query.get_scheduled_for(&db).await?;
@@ -2380,6 +2401,8 @@ pub async fn run_job_by_hash(
             concurrent_limit: concurrent_limit,
             concurrency_time_window_s: concurrency_time_window_s,
             cache_ttl,
+            language,
+            dedicated_worker,
         },
         args,
         &authed.username,

backend/windmill-api/src/lib.rs

@@ -59,6 +59,7 @@ mod saml;
 mod schedule;
 mod scim;
 mod scripts;
+mod settings;
 mod static_assets;
 mod tracing_init;
 mod users;
@@ -97,7 +98,7 @@ lazy_static::lazy_static! {
         .build().unwrap();
 
     pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
-        .map_err(|e| tracing::error!("Error building oauth clients: {}", e))
+        .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
         .unwrap();
 
     pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
@@ -155,7 +156,9 @@ pub async fn run_server(
         for tag in ALL_TAGS.clone() {
             let r =
                 rsmq_async::RsmqConnection::create_queue(&mut rsmq, &tag, None, None, None).await;
-            if r.is_ok() {
+            if let Err(e) = r {
+                tracing::info!("Redis queue {tag} could not be created: {e}");
+            } else {
                 tracing::info!("Redis queue {tag} created");
             }
         }
@@ -233,6 +236,7 @@ pub async fn run_server(
                     "/users",
                     users::global_service().layer(Extension(argon2.clone())),
                 )
+                .nest("/settings", settings::global_service())
                 .nest("/jobs", jobs::global_root_service())
                 .nest("/workers", workers::global_service())
                 .nest("/scripts", scripts::global_service())

backend/windmill-api/src/oauth2.rs

@@ -799,13 +799,20 @@ async fn slack_command(
                 (JobPayload::Flow(path.to_string()), None)
             } else {
                 let path = path.strip_prefix("script/").unwrap_or_else(|| path);
-                let (script_hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-                    windmill_common::get_latest_deployed_hash_for_path(
-                        &db,
-                        &settings.workspace_id,
-                        path,
-                    )
-                    .await?;
+                let (
+                    script_hash,
+                    tag,
+                    concurrent_limit,
+                    concurrency_time_window_s,
+                    cache_ttl,
+                    language,
+                    dedicated_worker,
+                ) = windmill_common::get_latest_deployed_hash_for_path(
+                    &db,
+                    &settings.workspace_id,
+                    path,
+                )
+                .await?;
                 (
                     JobPayload::ScriptHash {
                         hash: script_hash,
@@ -813,6 +820,8 @@ async fn slack_command(
                         concurrent_limit,
                         concurrency_time_window_s,
                         cache_ttl,
+                        language,
+                        dedicated_worker,
                     },
                     tag,
                 )

backend/windmill-api/src/resources.rs

@@ -31,6 +31,7 @@ use windmill_common::{
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
+        .route("/list_names/:type", get(list_names))
         .route("/get/*path", get(get_resource))
         .route("/exists/*path", get(exists_resource))
         .route("/get_value/*path", get(get_resource_value))
@@ -117,6 +118,32 @@ pub struct ListResourceQuery {
     resource_type: Option<String>,
     resource_type_exclude: Option<String>,
 }
+
+#[derive(Serialize, FromRow)]
+pub struct NamePath {
+    name: String,
+    path: String,
+}
+async fn list_names(
+    authed: ApiAuthed,
+    Path((w_id, rt)): Path<(String, String)>,
+    Extension(user_db): Extension<UserDB>,
+) -> JsonResult<Vec<NamePath>> {
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = sqlx::query!(
+        "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2",
+        rt,
+        &w_id
+    )
+    .fetch_all(&mut *tx)
+    .await?
+    .into_iter()
+    .filter_map(|x| x.name.map(|name| NamePath { name, path: x.path }))
+    .collect::<Vec<_>>();
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
 async fn list_resources(
     authed: ApiAuthed,
     Query(lq): Query<ListResourceQuery>,

backend/windmill-api/src/scripts.rs

@@ -71,6 +71,8 @@ pub struct ScriptWDraft {
     pub concurrency_time_window_s: Option<i32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_ttl: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub dedicated_worker: Option<bool>,
 }
 
 pub fn global_service() -> Router {
@@ -374,8 +376,8 @@ async fn create_script(
     sqlx::query!(
         "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
          content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
-         draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
+         draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)",
         &w_id,
         &hash.0,
         ns.path,
@@ -396,6 +398,7 @@ async fn create_script(
         ns.concurrent_limit,
         ns.concurrency_time_window_s,
         ns.cache_ttl,
+        ns.dedicated_worker
     )
     .execute(&mut tx)
     .await?;
@@ -592,7 +595,7 @@ async fn get_script_by_path_w_draft(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, ScriptWDraft>(
-        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, draft.value as draft FROM script LEFT JOIN draft ON 
+        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, draft.value as draft, dedicated_worker FROM script LEFT JOIN draft ON 
          script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
          WHERE script.path = $1 AND script.workspace_id = $2 \
          AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \

backend/windmill-api/src/settings.rs

@@ -0,0 +1,83 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2023
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    db::{ApiAuthed, DB},
+    utils::require_super_admin,
+};
+
+use axum::{
+    extract::{Extension, Path},
+    routing::{get, post},
+    Json, Router,
+};
+
+use windmill_common::{
+    error::{self, JsonResult},
+    global_settings::ENV_SETTINGS,
+};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/local", get(get_local_settings))
+        .route(
+            "/global/:key",
+            post(set_global_setting).get(get_global_setting),
+        )
+}
+
+pub async fn get_local_settings(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+) -> error::JsonResult<serde_json::Value> {
+    require_super_admin(&db, &authed.email).await?;
+
+    let mut settings = serde_json::Map::new();
+    for key in ENV_SETTINGS.iter() {
+        if let Some(value) = std::env::var(key).ok() {
+            settings.insert(key.to_string(), serde_json::Value::String(value));
+        }
+    }
+    Ok(Json(serde_json::Value::Object(settings)))
+}
+
+#[derive(serde::Deserialize)]
+pub struct Value {
+    pub value: serde_json::Value,
+}
+pub async fn set_global_setting(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Path(key): Path<String>,
+    Json(value): Json<Value>,
+) -> error::Result<()> {
+    require_super_admin(&db, &authed.email).await?;
+    sqlx::query!(
+        "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+        key,
+        value.value
+    )
+    .execute(&db)
+    .await?;
+    tracing::info!("Set global setting {} to {}", key, value.value);
+    Ok(())
+}
+
+pub async fn get_global_setting(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Path(key): Path<String>,
+) -> JsonResult<serde_json::Value> {
+    require_super_admin(&db, &authed.email).await?;
+    let value = sqlx::query!("SELECT value FROM global_settings WHERE name = $1", key)
+        .fetch_optional(&db)
+        .await?
+        .map(|x| x.value);
+
+    Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null)))
+}

backend/windmill-api/src/users.rs

@@ -717,7 +717,7 @@ async fn list_users(
                  WHERE workspace_id = $1
                    AND job_kind NOT IN ('flow', 'flowpreview')
                    AND email = usr.email
-                   AND now() - '5 week'::interval < created_at 
+                   AND now() - '1 week'::interval < created_at 
                ) usage
          WHERE workspace_id = $1
          ",

backend/windmill-api/src/workspaces.rs

@@ -252,7 +252,7 @@ async fn premium_info(
     require_admin(authed.is_admin, &authed.username)?;
     let mut tx = db.begin().await?;
     let row = sqlx::query_as::<_, PremiumWorkspaceInfo>(
-        "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = $1 AND usage.is_workspace IS true WHERE workspace.id = $1",
+        "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON usage.id = $1 AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND usage.is_workspace IS true WHERE workspace.id = $1",
     )
     .bind(w_id)
     .fetch_one(&mut *tx)
@@ -1515,7 +1515,7 @@ async fn tarball_workspace(
     if !skip_resources.unwrap_or(false) {
         let resources = sqlx::query_as!(
             Resource,
-            "SELECT * FROM resource WHERE workspace_id = $1",
+            "SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
             &w_id
         )
         .fetch_all(&db)

backend/windmill-common/Cargo.toml

@@ -16,6 +16,7 @@ tracing_init = [
     "dep:tracing",
     "dep:tracing-subscriber",
 ]
+flamegraph = ["dep:tracing-flame"]
 
 
 [lib]
@@ -41,4 +42,5 @@ hyper = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
 reqwest = { workspace = true, optional = true }
 tracing-subscriber = { workspace = true, optional = true }
-lazy_static.workspace = true
+lazy_static.workspace = true
+tracing-flame = { version = "^0", optional = true }

backend/windmill-common/src/flows.rs

@@ -297,6 +297,17 @@ pub enum FlowModuleValue {
     Identity,
 }
 
+impl FlowModuleValue {
+    pub fn is_simple(&self) -> bool {
+        match self {
+            FlowModuleValue::Script { .. } => true,
+            FlowModuleValue::Flow { .. } => true,
+            FlowModuleValue::RawScript { .. } => true,
+            _ => false,
+        }
+    }
+}
+
 fn ordered_map<S>(value: &HashMap<String, InputTransform>, serializer: S) -> Result<S::Ok, S::Error>
 where
     S: Serializer,

backend/windmill-common/src/global_settings.rs

@@ -0,0 +1,58 @@
+pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync";
+
+pub const ENV_SETTINGS: [&str; 54] = [
+    "DISABLE_NSJAIL",
+    "DISABLE_SERVER",
+    "NUM_WORKERS",
+    "METRICS_ADDR",
+    "JSON_FMT",
+    "BASE_URL",
+    "TIMEOUT",
+    "ZOMBIE_JOB_TIMEOUT",
+    "RESTART_ZOMBIE_JOBS",
+    "SLEEP_QUEUE",
+    "MAX_LOG_SIZE",
+    "SERVER_BIND_ADDR",
+    "PORT",
+    "KEEP_JOB_DIR",
+    "S3_CACHE_BUCKET",
+    "TAR_CACHE_RATE",
+    "COOKIE_DOMAIN",
+    "PYTHON_PATH",
+    "DENO_PATH",
+    "GO_PATH",
+    "GOPRIVATE",
+    "GOPROXY",
+    "NETRC",
+    "PIP_INDEX_URL",
+    "PIP_EXTRA_INDEX_URL",
+    "PIP_TRUSTED_HOST",
+    "PATH",
+    "HOME",
+    "DATABASE_CONNECTIONS",
+    "TIMEOUT_WAIT_RESULT",
+    "QUEUE_LIMIT_WAIT_RESULT",
+    "DENO_AUTH_TOKENS",
+    "DENO_FLAGS",
+    "NPM_CONFIG_REGISTRY",
+    "PIP_LOCAL_DEPENDENCIES",
+    "ADDITIONAL_PYTHON_PATHS",
+    "INCLUDE_HEADERS",
+    "INSTANCE_EVENTS_WEBHOOK",
+    "CLOUD_HOSTED",
+    "GLOBAL_CACHE_INTERVAL",
+    "WORKER_TAGS",
+    "CUSTOM_TAGS",
+    "JOB_RETENTION_SECS",
+    "WAIT_RESULT_FAST_POLL_DURATION_SECS",
+    "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
+    "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
+    "EXIT_AFTER_NO_JOB_FOR_SECS",
+    "REQUEST_SIZE_LIMIT",
+    "SMTP_HOST",
+    "SMTP_USERNAME",
+    "SMTP_PORT",
+    "SMTP_TLS_IMPLICIT",
+    "CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
+    "GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
+];

backend/windmill-common/src/jobs.rs

@@ -104,6 +104,14 @@ impl QueuedJob {
     pub fn is_flow(&self) -> bool {
         matches!(self.job_kind, JobKind::Flow | JobKind::FlowPreview)
     }
+
+    pub fn full_path(&self) -> String {
+        format!(
+            "{}/{}",
+            if self.is_flow() { "flow" } else { "script" },
+            self.script_path()
+        )
+    }
 }
 
 impl QueuedJob {
@@ -177,6 +185,8 @@ pub enum JobPayload {
         concurrent_limit: Option<i32>,
         concurrency_time_window_s: Option<i32>,
         cache_ttl: Option<i32>,
+        dedicated_worker: Option<bool>,
+        language: ScriptLang,
     },
     Code(RawCode),
     Dependencies {
@@ -224,8 +234,15 @@ pub async fn script_path_to_payload(
     let (job_payload, tag) = if script_path.starts_with("hub/") {
         (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
     } else {
-        let (script_hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-            get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
+        let (
+            script_hash,
+            tag,
+            concurrent_limit,
+            concurrency_time_window_s,
+            cache_ttl,
+            language,
+            dedicated_worker,
+        ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
         (
             JobPayload::ScriptHash {
                 hash: script_hash,
@@ -233,6 +250,8 @@ pub async fn script_path_to_payload(
                 concurrent_limit,
                 concurrency_time_window_s,
                 cache_ttl: cache_ttl,
+                language,
+                dedicated_worker,
             },
             tag,
         )
@@ -244,9 +263,16 @@ pub async fn script_hash_to_tag_and_limits<'c>(
     script_hash: &ScriptHash,
     db: &mut Transaction<'c, Postgres>,
     w_id: &String,
-) -> error::Result<(Option<Tag>, Option<i32>, Option<i32>, Option<i32>)> {
+) -> error::Result<(
+    Option<Tag>,
+    Option<i32>,
+    Option<i32>,
+    Option<i32>,
+    ScriptLang,
+    Option<bool>,
+)> {
     let script = sqlx::query!(
-        "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where hash = $1 AND workspace_id = $2",
+        "select tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where hash = $1 AND workspace_id = $2",
         script_hash.0,
         w_id
     )
@@ -262,6 +288,8 @@ pub async fn script_hash_to_tag_and_limits<'c>(
         script.concurrent_limit,
         script.concurrency_time_window_s,
         script.cache_ttl,
+        script.language,
+        script.dedicated_worker,
     ))
 }
 

backend/windmill-common/src/lib.rs

@@ -9,6 +9,7 @@
 use std::net::SocketAddr;
 
 use error::Error;
+use scripts::ScriptLang;
 use sqlx::{Pool, Postgres};
 
 pub mod apps;
@@ -17,6 +18,7 @@ pub mod error;
 pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
+pub mod global_settings;
 pub mod jobs;
 pub mod more_serde;
 pub mod oauth2;
@@ -169,9 +171,11 @@ pub async fn get_latest_deployed_hash_for_path(
     Option<i32>,
     Option<i32>,
     Option<i32>,
+    ScriptLang,
+    Option<bool>,
 )> {
     let r_o = sqlx::query!(
-        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND
+        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker from script where path = $1 AND workspace_id = $2 AND
     created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
     deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
         script_path,
@@ -188,6 +192,8 @@ pub async fn get_latest_deployed_hash_for_path(
         script.concurrent_limit,
         script.concurrency_time_window_s,
         script.cache_ttl,
+        script.language,
+        script.dedicated_worker,
     ))
 }
 
@@ -201,9 +207,11 @@ pub async fn get_latest_hash_for_path<'c>(
     Option<i32>,
     Option<i32>,
     Option<i32>,
+    ScriptLang,
+    Option<bool>,
 )> {
     let r_o = sqlx::query!(
-        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl from script where path = $1 AND workspace_id = $2 AND
+        "select hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker  from script where path = $1 AND workspace_id = $2 AND
     created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
     deleted = false AND archived = false)",
         script_path,
@@ -220,5 +228,7 @@ pub async fn get_latest_hash_for_path<'c>(
         script.concurrent_limit,
         script.concurrency_time_window_s,
         script.cache_ttl,
+        script.language,
+        script.dedicated_worker,
     ))
 }

backend/windmill-common/src/scripts.rs

@@ -215,6 +215,7 @@ pub struct NewScript {
     pub concurrent_limit: Option<i32>,
     pub concurrency_time_window_s: Option<i32>,
     pub cache_ttl: Option<i32>,
+    pub dedicated_worker: Option<bool>,
 }
 
 #[derive(Deserialize)]

backend/windmill-common/src/tracing_init.rs

@@ -48,3 +48,19 @@ pub fn initialize_tracing() {
             .init(),
     }
 }
+
+#[cfg(feature = "flamegraph")]
+use tracing_flame::FlameLayer;
+
+#[cfg(feature = "flamegraph")]
+pub fn setup_flamegraph() -> impl Drop {
+    // let fmt_layer = Layer::default();
+
+    let (flame_layer, _guard) = FlameLayer::with_file("./tracing.folded").unwrap();
+
+    tracing_subscriber::registry()
+        // .with(fmt_layer)
+        .with(flame_layer)
+        .init();
+    _guard
+}

backend/windmill-queue/Cargo.toml

@@ -11,6 +11,7 @@ path = "src/lib.rs"
 [features]
 default = []
 enterprise = []
+benchmark = []
 
 [dependencies]
 windmill-audit.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -8,6 +8,9 @@
 
 use std::{collections::HashMap, sync::atomic::AtomicBool, vec};
 
+#[cfg(feature = "benchmark")]
+use std::time::Instant;
+
 use anyhow::Context;
 use async_recursion::async_recursion;
 use chrono::{DateTime, Duration, Utc};
@@ -84,33 +87,36 @@ lazy_static::lazy_static! {
         "hub".to_string(),
         "other".to_string()];
 
-    pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
+    pub static ref DEDICATED_WORKER: Option<(String, String)> = std::env::var("DEDICATED_WORKER")
+        .ok()
+        .map(|x| {
+            let splitted = x.split(':').to_owned().collect_vec();
+            if splitted.len() != 2 {
+                panic!("DEDICATED_WORKER should be in the form of <workspace>:<script_path>")
+            } else {
+                let workspace = splitted[0];
+                let script_path = splitted[1];
+                (workspace.to_string(), script_path.to_string())
+            }
+        });
+
+    pub static ref ACCEPTED_TAGS: Vec<String> = {
+        let worker_tags = std::env::var("WORKER_TAGS")
         .ok()
         .map(|x| x.split(',').map(|x| x.to_string()).collect())
-        .unwrap_or_else(|| DEFAULT_TAGS.clone()) ;
+        .unwrap_or_else(|| DEFAULT_TAGS.clone());
+        if let Some(ref dedicated_worker) = DEDICATED_WORKER.as_ref() {
+            vec![format!("{}:{}", dedicated_worker.0, dedicated_worker.1)]
+        } else {
+            worker_tags
+         }
+    };
 
     pub static ref IS_WORKER_TAGS_DEFINED: bool = std::env::var("WORKER_TAGS").ok().is_some();
 
-    pub static ref PULL_QUERY: String = format!(
-        "UPDATE queue
-        SET running = true
-          , started_at = coalesce(started_at, now())
-          , last_ping = now()
-          , suspend_until = null
-        WHERE id = (
-            SELECT id
-            FROM queue
-            WHERE ((running = false AND scheduled_for <= now())
-               OR (suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()))) 
-                {}
-            ORDER BY scheduled_for, created_at
-            FOR UPDATE SKIP LOCKED
-            LIMIT 1
-        )
-        RETURNING *",
-        format!(" AND ({})",
-        ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "))
-    );
+
+
+
 
     // When compiled in 'benchmark' mode, this flags is exposed via the /workers/toggle endpoint
     // and make it possible to disable to current active workers (such that they don't pull any)
@@ -161,11 +167,12 @@ pub async fn cancel_job<'c: 'async_recursion>(
         let reason = reason
             .clone()
             .unwrap_or_else(|| "No reason provided".to_string());
+        let e = serde_json::json!({"message": format!("Job canceled: {reason} by {username}"), "name": "Canceled", "reason": reason, "canceler": username});
         let add_job = add_completed_job_error(
             &db,
             &job_running,
             format!("canceled by {username}: (force cancel: {force_cancel})"),
-            serde_json::json!({"message": format!("Job canceled: {reason} by {username}"), "name": "Canceled", "reason": reason, "canceler": username}),
+            &e,
             None,
             rsmq.clone(),
         )
@@ -216,7 +223,7 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     logs: String,
-    e: serde_json::Value,
+    e: &serde_json::Value,
     metrics: Option<Metrics>,
     rsmq: Option<R>,
 ) -> Result<serde_json::Value, Error> {
@@ -224,7 +231,7 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
         metrics.map(|m| m.worker_execution_failed.inc());
     }
     let result = serde_json::json!({ "error": e });
-    let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs, rsmq).await?;
+    let _ = add_completed_job(db, &queued_job, false, false, &result, logs, rsmq).await?;
     Ok(result)
 }
 
@@ -250,13 +257,13 @@ lazy_static::lazy_static! {
     pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
 }
 
-#[instrument(level = "trace", skip_all)]
+#[instrument(level = "trace", skip_all, name = "add_completed_job")]
 pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
     db: &Pool<Postgres>,
     queued_job: &QueuedJob,
     success: bool,
     skipped: bool,
-    result: serde_json::Value,
+    result: &serde_json::Value,
     logs: String,
     rsmq: Option<R>,
 ) -> Result<Uuid, Error> {
@@ -399,6 +406,19 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         )
         .await?;
     }
+    if queued_job.concurrent_limit.is_some() {
+        if let Err(e) = sqlx::query_scalar!(
+            "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1",
+            queued_job.full_path()
+        )
+        .execute(&mut tx)
+        .await
+        {
+            tracing::error!("Could not decrement concurrency counter: {}", e);
+        }
+        tracing::debug!("decremented concurrency counter");
+    }
+
     tx.commit().await?;
 
     #[cfg(feature = "enterprise")]
@@ -940,14 +960,15 @@ async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + '
 pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
     db: &Pool<Postgres>,
     rsmq: Option<R>,
+    suspend_first: bool,
 ) -> windmill_common::error::Result<Option<QueuedJob>> {
-    // let rs = rd_string(2);
-    // let instant = Instant::now();
-
     loop {
-        let tx: QueueTransaction<'_, _> = (rsmq.clone(), db.clone().begin().await?).into();
-        let (job, mut tx) =
-            pull_single_job_and_mark_as_running_no_concurrency_limit(tx, rsmq.clone()).await?;
+        let job = pull_single_job_and_mark_as_running_no_concurrency_limit(
+            db,
+            rsmq.clone(),
+            suspend_first,
+        )
+        .await?;
 
         if job.is_none() {
             return Ok(None);
@@ -962,10 +983,13 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             if *METRICS_ENABLED {
                 QUEUE_PULL_COUNT.inc();
             }
-            tx.commit().await?;
             return Ok(Option::Some(pulled_job));
         }
 
+        let itx = db.begin().await?;
+
+        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), itx).into();
+
         // Else the job is subject to concurrency limits
         let job_script_path = pulled_job.script_path.clone().unwrap();
 
@@ -979,15 +1003,31 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             job_custom_concurrency_time_window_s
         );
 
+        let running_job = sqlx::query_scalar!(
+            "INSERT INTO concurrency_counter VALUES ($1, 1)
+        ON CONFLICT (concurrency_id) 
+        DO UPDATE SET counter = concurrency_counter.counter + 1
+        RETURNING concurrency_counter.counter",
+            pulled_job.full_path(),
+        )
+        .fetch_one(&mut tx)
+        .await
+        .map_err(|e| {
+            Error::InternalErr(format!(
+                "Error getting concurrency count for script path {job_script_path}: {e}"
+            ))
+        })?;
+        tracing::debug!("running_job: {}", running_job);
+
         let script_path_live_stats = sqlx::query!(
-            "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count
+            "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) AS completed_count
             FROM
                 (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count
                 FROM completed_job
                 WHERE script_path = $1 AND job_kind != 'dependencies' AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3 AND canceled = false
                 GROUP BY script_path) as j
             FULL OUTER JOIN
-                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count
+                (SELECT script_path, MIN(started_at) as min_started_at
                 FROM queue
                 WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $3 AND canceled = false
                 GROUP BY script_path) as q
@@ -1004,20 +1044,31 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
             ))
         })?;
 
-        let concurrent_jobs_for_this_script: Option<i64> = script_path_live_stats.total_count;
+        let concurrent_jobs_for_this_script =
+            script_path_live_stats.completed_count.unwrap_or_default() as i32 + running_job;
         tracing::debug!(
             "Current concurrent jobs for this script: {}",
-            concurrent_jobs_for_this_script.unwrap_or(-1)
+            concurrent_jobs_for_this_script
         );
-        if concurrent_jobs_for_this_script.is_none()
-            || concurrent_jobs_for_this_script.unwrap() <= i64::from(job_custom_concurrent_limit)
-        {
+        if concurrent_jobs_for_this_script <= job_custom_concurrent_limit {
             if *METRICS_ENABLED {
                 QUEUE_PULL_COUNT.inc();
             }
             tx.commit().await?;
             return Ok(Option::Some(pulled_job));
         }
+        let x = sqlx::query_scalar!(
+            "UPDATE concurrency_counter SET counter = counter - 1 WHERE concurrency_id = $1 RETURNING counter",
+            pulled_job.full_path()
+        )
+        .fetch_one(&mut tx)
+        .await
+        .map_err(|e| {
+            Error::InternalErr(format!(
+                "Error decreasing concurrency count for script path {job_script_path}: {e}"
+            ))
+        })?;
+        tracing::debug!("running_job after decrease: {}", x);
 
         let job_uuid: Uuid = pulled_job.id;
         let min_started_at: Option<DateTime<Utc>> = script_path_live_stats.min_started_at;
@@ -1091,24 +1142,35 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
     'c,
     R: rsmq_async::RsmqConnection + Send + Clone,
 >(
-    mut tx: QueueTransaction<'c, R>,
+    db: &Pool<Postgres>,
     rsmq: Option<R>,
-) -> windmill_common::error::Result<(Option<QueuedJob>, QueueTransaction<'c, R>)> {
+    suspend_first: bool,
+) -> windmill_common::error::Result<Option<QueuedJob>> {
     let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
+        #[cfg(feature = "benchmark")]
+        let instant = Instant::now();
+
         // TODO: REDIS: Race conditions / replace last_ping
 
         // TODO: shuffle this list to have fairness
         let mut all_tags = ACCEPTED_TAGS.clone();
 
         let mut msg: Option<_> = None;
+        let mut tag = None;
+
         while msg.is_none() && !all_tags.is_empty() {
+            let ntag = all_tags.pop().unwrap();
+            tag = Some(ntag.clone());
             msg = rsmq
-                .pop_message::<Vec<u8>>(&all_tags.pop().unwrap())
+                .receive_message::<Vec<u8>>(&ntag, Some(10))
                 .await
                 .map_err(|e| anyhow::anyhow!(e))?;
         }
-        // println!("3.1: {:?} {rs}", instant.elapsed());
 
+        // #[cfg(feature = "benchmark")]
+        // println!("rsmq 1: {:?}", instant.elapsed());
+
+        // println!("3.1: {:?} {rs}", instant.elapsed());
         if let Some(msg) = msg {
             let uuid = Uuid::from_bytes_le(
                 msg.message
@@ -1116,7 +1178,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
                     .map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
             );
 
-            sqlx::query_as::<_, QueuedJob>(
+            let m2 = sqlx::query_as::<_, QueuedJob>(
                 "UPDATE queue
             SET running = true
             , started_at = coalesce(started_at, now())
@@ -1126,8 +1188,17 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
             RETURNING *",
             )
             .bind(uuid)
-            .fetch_optional(&mut tx)
-            .await?
+            .fetch_optional(db)
+            .await?;
+
+            rsmq.delete_message(&tag.unwrap(), &msg.id)
+                .await
+                .map_err(|e| anyhow::anyhow!(e))?;
+
+            #[cfg(feature = "benchmark")]
+            println!("rsmq 2: {:?}", instant.elapsed());
+
+            m2
         } else {
             None
         }
@@ -1139,11 +1210,62 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
          *   suspend_until is non-null
          *   and suspend = 0 when the resume messages are received
          *   or suspend_until <= now() if it has timed out */
-        sqlx::query_as::<_, QueuedJob>(&PULL_QUERY)
-            .fetch_optional(&mut tx)
-            .await?
+
+        let r = if suspend_first {
+            sqlx::query_as::<_, QueuedJob>("UPDATE queue
+            SET running = true
+              , started_at = coalesce(started_at, now())
+              , last_ping = now()
+              , suspend_until = null
+            WHERE id = (
+                SELECT id
+                FROM queue
+                WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag = ANY($1)
+                ORDER BY created_at
+                FOR UPDATE SKIP LOCKED
+                LIMIT 1
+            )
+            RETURNING *")
+                .bind(ACCEPTED_TAGS.as_slice())
+                .fetch_optional(db)
+                .await?
+        } else {
+            None
+        };
+
+        if r.is_none() {
+            // #[cfg(feature = "benchmark")]
+            // let instant = Instant::now();
+
+            let r = sqlx::query_as::<_, QueuedJob>(
+                "UPDATE queue
+            SET running = true
+              , started_at = coalesce(started_at, now())
+              , last_ping = now()
+              , suspend_until = null
+            WHERE id = (
+                SELECT id
+                FROM queue
+                WHERE running = false AND scheduled_for <= now() AND tag = ANY($1)
+                ORDER BY scheduled_for, created_at
+                FOR UPDATE SKIP LOCKED
+                LIMIT 1
+            )
+            RETURNING *",
+            )
+            .bind(ACCEPTED_TAGS.as_slice())
+            .fetch_optional(db)
+            .await?;
+
+            // #[cfg(feature = "benchmark")]
+            // println!("pull query: {:?}", instant.elapsed());
+
+            r
+        } else {
+            r
+        }
     };
-    Ok((job, tx))
+    Ok(job)
 }
 
 #[async_recursion]
@@ -1410,6 +1532,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         concurrent_limit,
         concurrency_time_window_s,
         cache_ttl,
+        dedicated_worker,
     ) = match job_payload {
         JobPayload::ScriptHash {
             hash,
@@ -1417,30 +1540,20 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             concurrent_limit,
             concurrency_time_window_s,
             cache_ttl,
-        } => {
-            let language = fetch_scalar_isolated!(sqlx::query_scalar!(
-                    "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND workspace_id = $2",
-                    hash.0,
-                    workspace_id
-                ), tx)
-                .ok().flatten()
-                .ok_or_else(||{
-                    Error::InternalErr(format!(
-                        "fetching language for hash {hash} in {workspace_id}"
-                    ))
-                })?;
-            (
-                Some(hash.0),
-                Some(path),
-                None,
-                JobKind::Script,
-                None,
-                Some(language),
-                concurrent_limit,
-                concurrency_time_window_s,
-                cache_ttl,
-            )
-        }
+            language,
+            dedicated_worker,
+        } => (
+            Some(hash.0),
+            Some(path),
+            None,
+            JobKind::Script,
+            None,
+            Some(language),
+            concurrent_limit,
+            concurrency_time_window_s,
+            cache_ttl,
+            dedicated_worker,
+        ),
         JobPayload::ScriptHub { path } => {
             (
                 None,
@@ -1453,6 +1566,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 None,
                 None,
                 None,
+                None,
             )
         }
         JobPayload::Code(RawCode {
@@ -1473,6 +1587,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             concurrent_limit,
             concurrency_time_window_s,
             cache_ttl,
+            None,
         ),
         JobPayload::Dependencies { hash, dependencies, language, path } => (
             Some(hash.0),
@@ -1484,6 +1599,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             None,
             None,
             None,
+            None,
         ),
         JobPayload::FlowDependencies { path } => {
             let value_json = fetch_scalar_isolated!(
@@ -1510,6 +1626,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 None,
                 None,
                 None,
+                None,
             )
         }
         JobPayload::AppDependencies { path, version } => (
@@ -1522,6 +1639,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             None,
             None,
             None,
+            None,
         ),
         JobPayload::RawFlow { value, path } => (
             None,
@@ -1533,6 +1651,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             value.concurrent_limit.clone(),
             value.concurrency_time_window_s,
             value.cache_ttl.map(|x| x as i32),
+            None,
         ),
         JobPayload::Flow(flow) => {
             let value_json = fetch_scalar_isolated!(
@@ -1559,6 +1678,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 value.concurrent_limit.clone(),
                 value.concurrency_time_window_s,
                 value.cache_ttl.map(|x| x as i32),
+                None,
             )
         }
         JobPayload::Identity => (
@@ -1571,6 +1691,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             None,
             None,
             None,
+            None,
         ),
         JobPayload::Noop => (
             None,
@@ -1582,6 +1703,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             None,
             None,
             None,
+            None,
         ),
     };
 
@@ -1636,7 +1758,13 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
 
-    let tag = if job_kind == JobKind::Script_Hub {
+    let tag = if dedicated_worker.is_some_and(|x| x) {
+        format!(
+            "{}:{}",
+            workspace_id,
+            script_path.clone().expect("dedicated script has a path")
+        )
+    } else if job_kind == JobKind::Script_Hub {
         "hub".to_string()
     } else {
         if tag == Some("".to_string()) {
@@ -1654,12 +1782,13 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 "deno".to_string()
             }
         };
-        tag.unwrap_or_else(|| {
-            language
-                .as_ref()
-                .map(|x| x.as_str().replace("$workspace", workspace_id))
-                .unwrap_or_else(default)
-        })
+        tag.map(|x| x.as_str().replace("$workspace", workspace_id).to_string())
+            .unwrap_or_else(|| {
+                language
+                    .as_ref()
+                    .map(|x| x.as_str().to_string())
+                    .unwrap_or_else(default)
+            })
     };
 
     let mut tx = match tx {

backend/windmill-queue/src/schedule.rs

@@ -74,13 +74,20 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     let (payload, tag) = if schedule.is_flow {
         (JobPayload::Flow(schedule.script_path), None)
     } else {
-        let (hash, tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-            windmill_common::get_latest_hash_for_path(
-                tx.transaction_mut(),
-                &schedule.workspace_id,
-                &schedule.script_path,
-            )
-            .await?;
+        let (
+            hash,
+            tag,
+            concurrent_limit,
+            concurrency_time_window_s,
+            cache_ttl,
+            language,
+            dedicated_worker,
+        ) = windmill_common::get_latest_hash_for_path(
+            tx.transaction_mut(),
+            &schedule.workspace_id,
+            &schedule.script_path,
+        )
+        .await?;
         (
             JobPayload::ScriptHash {
                 hash,
@@ -88,6 +95,8 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 concurrent_limit: concurrent_limit,
                 concurrency_time_window_s: concurrency_time_window_s,
                 cache_ttl: cache_ttl,
+                dedicated_worker,
+                language,
             },
             tag,
         )

backend/windmill-worker/Cargo.toml

@@ -11,6 +11,8 @@ path = "src/lib.rs"
 [features]
 default = []
 enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2"]
+benchmark = ["windmill-queue/benchmark"]
+flamegraph = []
 
 [dependencies]
 windmill-queue.workspace = true

backend/windmill-worker/loader.bun.ts

@@ -1,4 +1,4 @@
-const p: BunPlugin = {
+const p = {
   name: "windmill-relative-resolver",
   async setup(build) {
     const { writeFileSync, readFileSync, mkdirSync } = await import("fs");
@@ -41,11 +41,3 @@ const p: BunPlugin = {
     });
   },
 };
-
-import { BunPlugin } from "bun";
-
-Bun.build({
-  entrypoints: ["main.ts"],
-  outdir: "./out",
-  plugins: [p],
-});

backend/windmill-worker/src/bash_executor.rs

@@ -14,6 +14,7 @@ use crate::{
         transform_json_value, write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    TZ_ENV,
 };
 
 lazy_static::lazy_static! {
@@ -226,6 +227,7 @@ pub async fn handle_powershell_job(
             .current_dir(job_dir)
             .env_clear()
             .envs(reserved_variables)
+            .env("TZ", TZ_ENV.as_str())
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(cmd_args)
@@ -239,6 +241,7 @@ pub async fn handle_powershell_job(
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
+            .env("TZ", TZ_ENV.as_str())
             .env("PATH", PATH_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("HOME", HOME_ENV.as_str())

backend/windmill-worker/src/bigquery_executor.rs

@@ -6,7 +6,7 @@ use windmill_queue::HTTP_CLIENT;
 
 use serde::Deserialize;
 
-use crate::{common::transform_json_value, AuthedClient, JobCompleted};
+use crate::{common::transform_json_value, AuthedClient};
 
 use gcp_auth::{AuthenticationManager, CustomServiceAccount};
 
@@ -55,7 +55,7 @@ pub async fn do_bigquery(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
-) -> windmill_common::error::Result<JobCompleted> {
+) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
         Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
     } else {
@@ -176,12 +176,7 @@ pub async fn do_bigquery(
             }
 
             if result.rows.is_none() || result.rows.as_ref().unwrap().len() == 0 {
-                return Ok(JobCompleted {
-                    job: job,
-                    result: Value::Array(vec![]),
-                    logs: "".to_string(),
-                    success: true,
-                });
+                return Ok(Value::Array(vec![]));
             }
 
             if result.schema.is_none() {
@@ -223,12 +218,7 @@ pub async fn do_bigquery(
                 })
                 .collect();
 
-            return Ok(JobCompleted {
-                job: job,
-                result: rows,
-                logs: "".to_string(),
-                success: true,
-            });
+            return Ok(rows);
         }
         Err(e) => match response.json::<BigqueryErrorResponse>().await {
             Ok(bq_err) => return Err(Error::ExecutionErr(bq_err.error.message)),

backend/windmill-worker/src/bun_executor.rs

@@ -1,21 +1,45 @@
 use std::{collections::HashMap, process::Stdio};
 
+#[cfg(feature = "enterprise")]
+use std::collections::VecDeque;
+
+#[cfg(feature = "enterprise")]
+use anyhow::Context;
+
 use base64::Engine;
 use itertools::Itertools;
 use uuid::Uuid;
 
+#[cfg(feature = "enterprise")]
+use crate::{common::build_envs_map, JobCompleted};
+
 use crate::{
     common::{
         create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
         write_file, write_file_binary,
     },
     AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
-    NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV,
+    NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
-use tokio::{fs::File, io::AsyncReadExt, process::Command};
-use windmill_common::error::Result;
+
+#[cfg(feature = "enterprise")]
+use crate::MAX_BUFFERED_DEDICATED_JOBS;
+
+use tokio::{fs::File, process::Command};
+
+#[cfg(feature = "enterprise")]
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+
+use tokio::io::AsyncReadExt;
+
+#[cfg(feature = "enterprise")]
+use tokio::sync::mpsc::{Receiver, Sender};
+
+#[cfg(feature = "enterprise")]
+use windmill_common::variables;
+
 use windmill_common::{
-    error::{self},
+    error::{self, Result},
     jobs::QueuedJob,
 };
 use windmill_parser::Typ;
@@ -26,8 +50,8 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.ts");
 
 const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
 
-const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
-const EMPTY_FILE: &str = "<empty>";
+pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
+pub const EMPTY_FILE: &str = "<empty>";
 
 pub async fn gen_lockfile(
     logs: &mut String,
@@ -71,8 +95,6 @@ pub async fn gen_lockfile(
         .stderr(Stdio::piped())
         .spawn()?;
 
-    // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
-    // start = Instant::now();
     handle_child(
         job_id,
         db,
@@ -139,8 +161,6 @@ pub async fn install_lockfile(
         .stderr(Stdio::piped())
         .spawn()?;
 
-    // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
-    // start = Instant::now();
     handle_child(
         job_id,
         db,
@@ -170,8 +190,6 @@ pub async fn handle_bun_job(
     envs: HashMap<String, String>,
     shared_mount: &str,
 ) -> error::Result<serde_json::Value> {
-    set_logs(&logs, &job.id, &db).await;
-
     let _ = write_file(job_dir, "main.ts", inner_content).await?;
     let common_bun_proc_envs: HashMap<String, String> =
         get_common_bun_proc_envs(&base_internal_url);
@@ -209,6 +227,7 @@ pub async fn handle_bun_job(
         .await?;
     } else if !*DISABLE_NSJAIL {
         logs.push_str("\n\n--- BUN INSTALL ---\n");
+        set_logs(&logs, &job.id, &db).await;
         let _ = gen_lockfile(
             logs,
             &job.id,
@@ -224,9 +243,12 @@ pub async fn handle_bun_job(
         .await?;
     }
 
-    // let mut start = Instant::now();
     logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
-    set_logs(&logs, &job.id, &db).await;
+
+    let logs_f = async {
+        set_logs(&logs, &job.id, &db).await;
+        Ok(()) as error::Result<()>
+    };
 
     let write_wrapper_f = async {
         // let mut start = Instant::now();
@@ -291,27 +313,35 @@ run().catch(async (e) => {{
         Ok(reserved_variables) as error::Result<HashMap<String, String>>
     };
 
-    let (reserved_variables, _) = tokio::try_join!(reserved_variables_args_out_f, write_wrapper_f)?;
-
-    let _ = write_file(
-        &job_dir,
-        "loader.bun.ts",
-        &format!(
-            r#"
+    let write_loader_f = async {
+        write_file(
+            &job_dir,
+            "loader.bun.ts",
+            &format!(
+                r#"
 import {{ plugin }} from "bun";
 
 {}
 
 plugin(p)
 "#,
-            RELATIVE_BUN_LOADER
-                .replace("W_ID", &job.workspace_id)
-                .replace("BASE_INTERNAL_URL", base_internal_url)
-                .replace("TOKEN", &client.get_token().await)
-                .replace("CURRENT_PATH", job.script_path())
-        ),
-    )
-    .await?;
+                RELATIVE_BUN_LOADER
+                    .replace("W_ID", &job.workspace_id)
+                    .replace("BASE_INTERNAL_URL", base_internal_url)
+                    .replace("TOKEN", &client.get_token().await)
+                    .replace("CURRENT_PATH", job.script_path())
+            ),
+        )
+        .await?;
+        Ok(()) as error::Result<()>
+    };
+
+    let (reserved_variables, _, _, _) = tokio::try_join!(
+        reserved_variables_args_out_f,
+        write_wrapper_f,
+        logs_f,
+        write_loader_f
+    )?;
 
     //do not cache local dependencies
     let child = if !*DISABLE_NSJAIL {
@@ -370,8 +400,6 @@ plugin(p)
             .spawn()?
     };
 
-    // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
-    // start = Instant::now();
     handle_child(
         &job.id,
         db,
@@ -384,13 +412,13 @@ plugin(p)
         job.timeout,
     )
     .await?;
-    // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
     read_result(job_dir).await
 }
 
-fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
+pub fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
     let mut deno_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("TZ"), TZ_ENV.clone()),
         (String::from("DISABLE_COLORS"), "0".to_string()),
         (String::from("DO_NOT_TRACK"), "1".to_string()),
         (
@@ -410,3 +438,298 @@ fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String>
     }
     return deno_envs;
 }
+
+#[cfg(feature = "enterprise")]
+pub async fn start_worker(
+    requirements_o: Option<String>,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    inner_content: &str,
+    base_internal_url: &str,
+    job_dir: &str,
+    worker_name: &str,
+    envs: HashMap<String, String>,
+    w_id: &str,
+    script_path: &str,
+    token: &str,
+    job_completed_tx: Sender<JobCompleted>,
+    mut jobs_rx: Receiver<QueuedJob>,
+    mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
+) -> Result<()> {
+    let mut logs = "".to_string();
+    let _ = write_file(job_dir, "main.ts", inner_content).await?;
+    let common_bun_proc_envs: HashMap<String, String> =
+        get_common_bun_proc_envs(&base_internal_url);
+    let context = variables::get_reserved_variables(
+        w_id,
+        &token,
+        "dedicated_worker@windmill.dev",
+        "dedicated_worker",
+        "NOT_AVAILABLE",
+        "dedicated_worker",
+        Some(script_path.to_string()),
+        None,
+        None,
+        None,
+        None,
+    )
+    .to_vec();
+    let context_envs = build_envs_map(context);
+    if let Some(reqs) = requirements_o {
+        let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
+        if splitted.len() != 2 {
+            return Err(error::Error::ExecutionErr(
+                format!("Invalid requirements, expectd to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
+            ));
+        }
+        let _ = write_file(job_dir, "package.json", &splitted[0]).await?;
+        let lockb = splitted[1];
+        if lockb != EMPTY_FILE {
+            let _ = write_file_binary(
+                job_dir,
+                "bun.lockb",
+                &base64::engine::general_purpose::STANDARD
+                    .decode(&splitted[1])
+                    .map_err(|_| {
+                        error::Error::InternalErr("Could not decode bun.lockb".to_string())
+                    })?,
+            )
+            .await?;
+        }
+        install_lockfile(
+            &mut logs,
+            &Uuid::nil(),
+            &w_id,
+            db,
+            job_dir,
+            worker_name,
+            common_bun_proc_envs.clone(),
+        )
+        .await?;
+    } else if !*DISABLE_NSJAIL {
+        logs.push_str("\n\n--- BUN INSTALL ---\n");
+        let _ = gen_lockfile(
+            &mut logs,
+            &Uuid::nil(),
+            &w_id,
+            db,
+            token,
+            &script_path,
+            job_dir,
+            base_internal_url,
+            worker_name,
+            false,
+        )
+        .await?;
+    }
+
+    {
+        // let mut start = Instant::now();
+        let args = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args;
+        let dates = args
+            .iter()
+            .enumerate()
+            .filter_map(|(i, x)| {
+                if matches!(x.typ, Typ::Datetime) {
+                    Some(i)
+                } else {
+                    None
+                }
+            })
+            .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
+            .join("\n");
+
+        let spread = args.into_iter().map(|x| x.name).join(",");
+        // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
+        // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud
+        let wrapper_content: String = format!(
+            r#"
+import {{ main }} from "./main.ts";
+
+BigInt.prototype.toJSON = function () {{
+    return this.toString();
+}};
+
+{dates}
+
+let stdout = Bun.stdout.writer();
+// let stdout = Bun.file("output.txt").writer();
+stdout.write('start\n'); 
+
+for await (const chunk of Bun.stdin.stream()) {{
+    const lines = Buffer.from(chunk).toString();
+    let exit = false;
+    for (const line of lines.trim().split("\n")) {{
+        // stdout.write('s: ' + line + 'EE\n'); 
+        if (line === "end") {{
+            exit = true;
+            break;
+        }}
+        try {{
+            let {{ {spread} }} = JSON.parse(line) 
+            let res: any = await main(...[ {spread} ]);
+            stdout.write(JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n');
+        }} catch (e) {{
+            stdout.write(JSON.stringify({{ error: {{ message: e.message, name: e.name, stack: e.stack, line: line }}}}) + '\n');
+        }}
+        stdout.flush();
+    }}
+    if (exit) {{
+        break;
+    }}
+}}
+"#,
+        );
+        write_file(job_dir, "wrapper.ts", &wrapper_content).await?;
+    }
+
+    let reserved_variables = windmill_common::variables::get_reserved_variables(
+        w_id,
+        token,
+        "dedicated_worker",
+        "dedicated_worker",
+        Uuid::nil().to_string().as_str(),
+        "dedicted_worker",
+        Some(script_path.to_string()),
+        None,
+        None,
+        None,
+        None,
+    );
+
+    let _ = write_file(
+        &job_dir,
+        "loader.bun.ts",
+        &format!(
+            r#"
+import {{ plugin }} from "bun";
+
+{}
+
+plugin(p)
+"#,
+            RELATIVE_BUN_LOADER
+                .replace("W_ID", &w_id)
+                .replace("BASE_INTERNAL_URL", base_internal_url)
+                .replace("TOKEN", token)
+                .replace("CURRENT_PATH", script_path)
+        ),
+    )
+    .await?;
+
+    //do not cache local dependencies
+    let mut child = {
+        let script_path = format!("{job_dir}/wrapper.ts");
+        let args = vec![
+            "run",
+            "-i",
+            "--prefer-offline",
+            "-r",
+            "./loader.bun.ts",
+            &script_path,
+        ];
+        Command::new(&*BUN_PATH)
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(context_envs)
+            .envs(envs)
+            .envs(
+                reserved_variables
+                    .iter()
+                    .map(|x| (x.name.clone(), x.value.clone()))
+                    .collect::<Vec<_>>(),
+            )
+            .envs(common_bun_proc_envs)
+            .args(args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+
+    let stdout = child
+        .stdout
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let mut reader = BufReader::new(stdout).lines();
+
+    let mut stdin = child
+        .stdin
+        .take()
+        .expect("child did not have a handle to stdin");
+
+    // Ensure the child process is spawned in the runtime so it can
+    // make progress on its own while we await for any output.
+    let child = tokio::spawn(async move {
+        let status = child
+            .wait()
+            .await
+            .expect("child process encountered an error");
+
+        println!("child status was: {}", status);
+    });
+
+    let mut jobs = VecDeque::with_capacity(MAX_BUFFERED_DEDICATED_JOBS);
+    // let mut i = 0;
+    // let mut j = 0;
+    let mut alive = true;
+    loop {
+        tokio::select! {
+            biased;
+            _ = killpill_rx.recv(), if alive => {
+                println!("received killpill for dedicated worker");
+                alive = false;
+                if let Err(e) = write_stdin(&mut stdin, "end").await {
+                    tracing::info!("Could not write end message to stdin: {e:?}")
+                }
+            },
+            line = reader.next_line() => {
+                // j += 1;
+
+                if let Some(line) = line.expect("line is ok") {
+                    if line == "start" {
+                        tracing::info!("dedicated worker process started");
+                        continue;
+                    }
+                    tracing::debug!("processed job");
+
+                    let result = serde_json::from_str(&line).expect("json is ok");
+                    let job: QueuedJob = jobs.pop_front().expect("pop");
+                    job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), success: true, cached_res_path: None, token: token.to_string() }).await.unwrap();
+                } else {
+                    tracing::info!("dedicated worker process exited");
+                    break;
+                }
+            }
+            job = jobs_rx.recv(), if alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS => {
+                // i += 1;
+                if let Some(job) = job {
+                    tracing::debug!("received job");
+                    jobs.push_back(job.clone());
+                    // write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({"x": job.id}))).expect("serialize")).await?;
+                    write_stdin(&mut stdin, &serde_json::to_string(&job.args.unwrap_or_else(|| serde_json::json!({}))).expect("serialize")).await?;
+                    stdin.flush().await.context("stdin flush")?;
+                } else {
+                    tracing::debug!("job channel closed");
+                    alive = false;
+                    if let Err(e) = write_stdin(&mut stdin, "end").await {
+                        tracing::error!("Could not write end message to stdin: {e:?}")
+                    }
+                }
+            }
+        }
+    }
+
+    child
+        .await
+        .map_err(|e| anyhow::anyhow!("child process encountered an error: {e}"))?;
+    tracing::info!("dedicated worker child process exited successfully");
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+async fn write_stdin(stdin: &mut tokio::process::ChildStdin, s: &str) -> error::Result<()> {
+    let _ = &stdin.write_all(format!("{s}\n").as_bytes()).await?;
+    stdin.flush().await.context("stdin flush")?;
+    Ok(())
+}

backend/windmill-worker/src/common.rs

@@ -6,6 +6,7 @@ use windmill_api_client::{types::CreateResource, Client};
 use windmill_common::{
     error::{self, Error},
     jobs::QueuedJob,
+    variables::ContextualVariable,
 };
 use windmill_queue::CLOUD_HOSTED;
 
@@ -189,10 +190,12 @@ pub async fn get_reserved_variables(
     )
     .to_vec();
 
-    let mut r: HashMap<String, String> = variables
-        .into_iter()
-        .map(|rv| (rv.name, rv.value))
-        .collect();
+    Ok(build_envs_map(variables))
+}
+
+pub fn build_envs_map(context: Vec<ContextualVariable>) -> HashMap<String, String> {
+    let mut r: HashMap<String, String> =
+        context.into_iter().map(|rv| (rv.name, rv.value)).collect();
 
     if let Some(ref envs) = *WHITELIST_ENVS {
         for e in envs {
@@ -200,9 +203,8 @@ pub async fn get_reserved_variables(
         }
     }
 
-    Ok(r)
+    r
 }
-
 async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
     if pid.is_none() {
         return -1;
@@ -273,12 +275,15 @@ pub async fn handle_child(
     /* the cancellation future is polled on by `wait_on_child` while
      * waiting for the child to exit normally */
     let update_job = async {
+        if job_id == Uuid::nil() {
+            return;
+        }
         let db = db.clone();
 
         let mut interval = interval(update_job_interval);
         interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
 
-        let mut i = 1;
+        let mut i = 0;
         loop {
             tokio::select!(
                 _ = rx.recv() => break,
@@ -295,7 +300,7 @@ pub async fn handle_child(
                         .expect("update worker ping");
                     }
                     let mem_peak = get_mem_peak(pid, nsjail).await;
-                    tracing::info!("{job_id} still running. mem peak: {}kB", mem_peak);
+                    tracing::info!("{job_id} in {} still running. mem peak: {}kB", _w_id, mem_peak);
                     let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
                     if sqlx::query_scalar!("UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id)
                         .fetch_optional(&db)
@@ -357,7 +362,7 @@ pub async fn handle_child(
             result = child.wait() => return result.map(Ok),
             Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
             _ = sleep(timeout_duration) => KillReason::Timeout,
-            _ = update_job => KillReason::Cancelled,
+            _ = update_job, if job_id != Uuid::nil() => KillReason::Cancelled,
         };
         tx.send(()).expect("rx should never be dropped");
         drop(tx);
@@ -538,7 +543,7 @@ fn child_joined_output_stream(
     stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
 }
 
-fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
+pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
     mut lines: tokio::io::Lines<R>,
 ) -> impl futures::Stream<Item = io::Result<String>> {
     stream::poll_fn(move |cx| {

backend/windmill-worker/src/dedicated_worker.rs

@@ -0,0 +1,5 @@
+// use tokio::sync::mpsc;
+
+// pub fn create_dedicated_worker() {
+//     let (job_completed_tx, mut new_job) = mpsc::channel::<JobCompleted>(100);
+// }

backend/windmill-worker/src/deno_executor.rs

@@ -9,7 +9,7 @@ use crate::{
         write_file,
     },
     AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, NPM_CONFIG_REGISTRY,
-    PATH_ENV,
+    PATH_ENV, TZ_ENV,
 };
 use tokio::{fs::File, io::AsyncReadExt, process::Command};
 use windmill_common::{error::Result, BASE_URL};
@@ -50,6 +50,7 @@ fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<St
 
     let mut deno_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("TZ"), TZ_ENV.clone()),
         (String::from("DENO_AUTH_TOKENS"), deno_auth_tokens),
         (
             String::from("BASE_INTERNAL_URL"),

backend/windmill-worker/src/global_cache.rs

@@ -6,6 +6,11 @@ use itertools::Itertools;
 use rand::Rng;
 #[cfg(feature = "enterprise")]
 use std::process::Stdio;
+#[cfg(feature = "enterprise")]
+use windmill_common::DB;
+
+#[cfg(feature = "enterprise")]
+use windmill_common::global_settings::WORKER_S3_BUCKET_SYNC;
 
 #[cfg(feature = "enterprise")]
 use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
@@ -219,6 +224,25 @@ pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
     Ok(())
 }
 
+#[cfg(feature = "enterprise")]
+pub async fn worker_s3_bucket_sync_enabled(db: &DB) -> bool {
+    let q = sqlx::query!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        WORKER_S3_BUCKET_SYNC
+    )
+    .fetch_optional(db)
+    .await;
+
+    if let Ok(q) = q {
+        let r = q.map(|x| x.value.as_bool().unwrap_or(true)).unwrap_or(true);
+        tracing::info!("Got global setting {WORKER_S3_BUCKET_SYNC}: {}", r);
+        r
+    } else {
+        tracing::info!("Failed to get global setting {WORKER_S3_BUCKET_SYNC}");
+        false
+    }
+}
+
 #[cfg(feature = "enterprise")]
 pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
     tracing::info!("Copying cache to bucket {bucket} as tar");

backend/windmill-worker/src/go_executor.rs

@@ -12,7 +12,7 @@ use windmill_common::{
     jobs::QueuedJob,
     utils::calculate_hash,
 };
-use windmill_parser_go::parse_go_imports;
+use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE};
 
 use crate::{
     common::{
@@ -20,7 +20,7 @@ use crate::{
         set_logs, write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY,
-    GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
 
 const GO_REQ_SPLITTER: &str = "//go.sum\n";
@@ -214,8 +214,7 @@ func Run(req Req) (interface{{}}, error){{
         create_args_and_out_file(client, job, job_dir).await?;
     }
 
-    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
-    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
+    let reserved_variables = get_reserved_variables(job, &client.token, db).await?;
 
     let child = if !*DISABLE_NSJAIL {
         let _ = write_file(
@@ -234,6 +233,7 @@ func Run(req Req) (interface{{}}, error){{
             .envs(envs)
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
             .stdout(Stdio::piped())
@@ -247,6 +247,7 @@ func Run(req Req) (interface{{}}, error){{
             .envs(envs)
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .env("GOPATH", GO_CACHE_DIR)
             .env("HOME", HOME_ENV.as_str());
@@ -296,6 +297,9 @@ async fn gen_go_mod(
     }
 }
 
+use std::fs::OpenOptions;
+use std::io::prelude::*;
+
 pub async fn install_go_dependencies(
     job_id: &Uuid,
     code: &str,
@@ -329,6 +333,16 @@ pub async fn install_go_dependencies(
             None,
         )
         .await?;
+
+        for x in REQUIRE_PARSE.captures_iter(code) {
+            let mut file = OpenOptions::new()
+                .write(true)
+                .append(true)
+                .open(format!("{job_dir}/go.mod"))
+                .unwrap();
+
+            writeln!(file, "require {}\n", &x[1])?;
+        }
     }
 
     let mut new_lockfile = false;

backend/windmill-worker/src/graphql_executor.rs

@@ -7,7 +7,7 @@ use windmill_queue::HTTP_CLIENT;
 
 use serde::Deserialize;
 
-use crate::{common::transform_json_value, AuthedClient, JobCompleted};
+use crate::{common::transform_json_value, AuthedClient};
 
 #[derive(Deserialize)]
 struct GraphqlApi {
@@ -31,7 +31,7 @@ pub async fn do_graphql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
-) -> windmill_common::error::Result<JobCompleted> {
+) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
         Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
     } else {
@@ -88,10 +88,5 @@ pub async fn do_graphql(
     }
 
     // And then check that we got back the same string we sent over.
-    return Ok(JobCompleted {
-        job: job,
-        result: result.data.unwrap_or(json!({})),
-        logs: "".to_string(),
-        success: true,
-    });
+    return Ok(result.data.unwrap_or(json!({})));
 }

backend/windmill-worker/src/js_eval.rs

@@ -67,6 +67,7 @@ impl TimersPermission for PermissionsContainer {
         ()
     }
 }
+
 pub struct OptAuthedClient(Option<AuthedClient>);
 pub async fn eval_timeout(
     expr: String,
@@ -122,8 +123,6 @@ pub async fn eval_timeout(
                 .enable_all()
                 .build()?;
 
-            let re = Regex::new(r"import (.*)\n").unwrap();
-            let expr = re.replace_all(&expr, "").to_string();
             // pretty frail but this it to make the expr more user friendly and not require the user to write await
             let expr = ["variable", "step", "resource", "result_by_id"]
                 .into_iter()
@@ -442,7 +441,7 @@ pub async fn eval_fetch_timeout(
     let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
     let ts_expr2 = ts_expr.clone();
     timeout(
-        std::time::Duration::from_millis(100000),
+        std::time::Duration::from_secs(100),
         tokio::task::spawn_blocking(move || {
             let ops = vec![op_get_static_args::DECL, op_log::DECL];
             let ext = Extension {
@@ -537,7 +536,7 @@ pub async fn eval_fetch_timeout(
             isolate.terminate_execution();
         };
         Error::ExecutionErr(format!(
-            "The expression of evaluation `{ts_expr2}` took too long to execute (>10000ms)"
+            "The expression of evaluation `{ts_expr2}` took too long to execute (>100s)"
         ))
     })??
 }

backend/windmill-worker/src/lib.rs

@@ -6,6 +6,7 @@ mod snowflake_executor;
 mod bash_executor;
 mod bun_executor;
 mod common;
+mod dedicated_worker;
 mod deno_executor;
 mod global_cache;
 mod go_executor;

backend/windmill-worker/src/mysql_executor.rs

@@ -10,7 +10,7 @@ use windmill_common::{
 };
 use windmill_parser_sql::parse_mysql_sig;
 
-use crate::{common::transform_json_value, AuthedClient, JobCompleted};
+use crate::{common::transform_json_value, AuthedClient};
 
 #[derive(Deserialize)]
 struct MysqlDatabase {
@@ -26,7 +26,7 @@ pub async fn do_mysql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
-) -> windmill_common::error::Result<JobCompleted> {
+) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
         Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
     } else {
@@ -114,7 +114,7 @@ pub async fn do_mysql(
     pool.disconnect().await.map_err(to_anyhow)?;
 
     // And then check that we got back the same string we sent over.
-    return Ok(JobCompleted { job: job, result: json!(rows), logs: "".to_string(), success: true });
+    return Ok(json!(rows));
 }
 
 fn convert_row_to_value(row: Row) -> serde_json::Value {

backend/windmill-worker/src/pg_executor.rs

@@ -18,7 +18,7 @@ use windmill_common::{error::to_anyhow, jobs::QueuedJob};
 use windmill_parser_sql::parse_pgsql_sig;
 
 use crate::common::transform_json_value;
-use crate::{AuthedClient, JobCompleted};
+use crate::AuthedClient;
 use urlencoding::encode;
 
 #[derive(Deserialize)]
@@ -36,7 +36,7 @@ pub async fn do_postgresql(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
-) -> error::Result<JobCompleted> {
+) -> error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
         Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
     } else {
@@ -181,7 +181,7 @@ pub async fn do_postgresql(
 
     handle.abort();
     // And then check that we got back the same string we sent over.
-    return Ok(JobCompleted { job: job, result, logs: "".to_string(), success: true });
+    return Ok(result);
 }
 
 pub fn pg_cell_to_json_value(

backend/windmill-worker/src/python_executor.rs

@@ -62,7 +62,7 @@ use crate::{
         write_file,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HTTPS_PROXY, HTTP_PROXY,
-    LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR,
+    LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, TZ_ENV,
 };
 
 pub async fn create_dependencies_dir(job_dir: &str) {
@@ -280,8 +280,7 @@ pub async fn handle_python_job(
                 let name = &x.name;
                 format!(
                     "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n    \
-                                     kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], \
-                                     '%Y-%m-%dT%H:%M')\n",
+                                     kwargs[\"{name}\"] = datetime.fromisoformat(kwargs[\"{name}\"])\n",
                 )
             }
             _ => "".to_string(),
@@ -440,6 +439,7 @@ mount {{
             // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(vec![
                 "--config",
@@ -460,6 +460,7 @@ mount {{
             .envs(envs)
             .envs(reserved_variables)
             .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
             .args(vec!["-u", "-m", "wrapper"])
             .stdout(Stdio::piped())

backend/windmill-worker/src/snowflake_executor.rs

@@ -12,7 +12,7 @@ use windmill_queue::HTTP_CLIENT;
 
 use serde::{Deserialize, Serialize};
 
-use crate::{common::transform_json_value, AuthedClient, JobCompleted};
+use crate::{common::transform_json_value, AuthedClient};
 
 #[derive(Serialize)]
 struct Claims {
@@ -64,7 +64,7 @@ pub async fn do_snowflake(
     job: QueuedJob,
     client: &AuthedClient,
     query: &str,
-) -> windmill_common::error::Result<JobCompleted> {
+) -> windmill_common::error::Result<serde_json::Value> {
     let args = if let Some(args) = &job.args {
         Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
     } else {
@@ -201,7 +201,7 @@ pub async fn do_snowflake(
                 })
                 .collect();
 
-            Ok(JobCompleted { job: job, result: rows, logs: "".to_string(), success: true })
+            Ok(rows)
         }
         Err(e) => {
             let resp = response.text().await.unwrap_or("".to_string());

backend/windmill-worker/src/worker_flow.rs

@@ -54,7 +54,6 @@ pub async fn update_flow_status_after_job_completion<
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     stop_early_override: Option<bool>,
-    base_internal_url: &str,
     rsmq: Option<R>,
 ) -> error::Result<()> {
     // this is manual tailrecursion because async_recursion blows up the stack
@@ -71,7 +70,6 @@ pub async fn update_flow_status_after_job_completion<
         same_worker_tx.clone(),
         worker_dir,
         stop_early_override,
-        base_internal_url,
         rsmq.clone(),
     )
     .await?;
@@ -89,7 +87,6 @@ pub async fn update_flow_status_after_job_completion<
             same_worker_tx.clone(),
             worker_dir,
             nrec.stop_early_override,
-            base_internal_url,
             rsmq.clone(),
         )
         .await?;
@@ -119,20 +116,17 @@ pub async fn update_flow_status_after_job_completion_internal<
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
     stop_early_override: Option<bool>,
-    base_internal_url: &str,
     rsmq: Option<R>,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
     let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult) = {
         // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
 
-        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
-
         let old_status_json = sqlx::query_scalar!(
             "SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
             flow,
             w_id
         )
-        .fetch_one(&mut tx)
+        .fetch_one(db)
         .await
         .map_err(|e| {
             Error::InternalErr(format!(
@@ -161,12 +155,8 @@ pub async fn update_flow_status_after_job_completion_internal<
             module_status,
             FlowStatusModule::InProgress { iterator: Some(_), .. }
         ) {
-            let (loop_failures, parallelism) = compute_skip_loop_failures_and_parallelism(
-                flow,
-                old_status.step,
-                tx.transaction_mut(),
-            )
-            .await?;
+            let (loop_failures, parallelism) =
+                compute_skip_loop_failures_and_parallelism(flow, old_status.step, db).await?;
             (loop_failures.unwrap_or(false), parallelism)
         } else {
             (false, None)
@@ -190,7 +180,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             old_status.step,
             flow
         )
-        .fetch_one(&mut tx)
+        .fetch_one(db)
         .await
         .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?;
 
@@ -208,16 +198,16 @@ pub async fn update_flow_status_after_job_completion_internal<
             FlowStatusModule::InProgress {
                 branchall: Some(BranchAllStatus { branch, .. }),
                 ..
-            } => {
-                compute_skip_branchall_failure(flow, old_status.step, *branch, tx.transaction_mut())
-                    .await?
-                    .unwrap_or(false)
-            }
+            } => compute_skip_branchall_failure(flow, old_status.step, *branch, db)
+                .await?
+                .unwrap_or(false),
             _ => false,
         };
 
         let skip_failure = skip_branch_failure || skip_loop_failures;
 
+        let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
+
         let (inc_step_counter, new_status) = match module_status {
             FlowStatusModule::InProgress {
                 iterator,
@@ -278,6 +268,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                         .into_iter()
                         .all(|x| x)
                     {
+                        success = true;
                         FlowStatusModule::Success {
                             id: module_status.id(),
                             job: job_id_for_status.clone(),
@@ -575,7 +566,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                 db,
                 &flow_job,
                 logs,
-                canceled_job_to_result(&flow_job),
+                &canceled_job_to_result(&flow_job),
                 metrics.clone(),
                 rsmq.clone(),
             )
@@ -595,7 +586,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                 &flow_job,
                 success,
                 stop_early && skip_if_stop_early,
-                nresult.clone(),
+                &nresult,
                 logs,
                 rsmq.clone(),
             )
@@ -610,17 +601,17 @@ pub async fn update_flow_status_after_job_completion_internal<
             nresult.clone(),
             same_worker_tx.clone(),
             worker_dir,
-            base_internal_url,
             rsmq.clone(),
         )
         .await
         {
             Err(err) => {
+                let e = json!({"message": err.to_string(), "name": "InternalError"});
                 let _ = add_completed_job_error(
                     db,
                     &flow_job,
                     "Unexpected error during flow chaining:\n".to_string(),
-                    json!({"message": err.to_string(), "name": "InternalError"}),
+                    &e,
                     metrics.clone(),
                     rsmq.clone(),
                 )
@@ -655,10 +646,10 @@ pub async fn update_flow_status_after_job_completion_internal<
     }
 }
 
-async fn compute_skip_loop_failures_and_parallelism<'c>(
+async fn compute_skip_loop_failures_and_parallelism(
     flow: Uuid,
     step: i32,
-    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    db: &DB,
 ) -> Result<(Option<bool>, Option<i32>), Error> {
     sqlx::query_as(
         "
@@ -669,7 +660,7 @@ async fn compute_skip_loop_failures_and_parallelism<'c>(
     )
     .bind(step)
     .bind(flow)
-    .fetch_one(&mut **tx)
+    .fetch_one(db)
     .await
     .map(|(v, n)| (v,n))
     .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e}")))
@@ -679,7 +670,7 @@ async fn compute_skip_branchall_failure<'c>(
     flow: Uuid,
     step: i32,
     branch: usize,
-    tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    db: &DB,
 ) -> Result<Option<bool>, Error> {
     sqlx::query_as(
         "
@@ -691,7 +682,7 @@ async fn compute_skip_branchall_failure<'c>(
     .bind(step)
     .bind(branch as i32)
     .bind(flow)
-    .fetch_one(&mut **tx)
+    .fetch_one(db)
     .await
     .map(|(v,)| v)
     .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e}")))
@@ -885,7 +876,6 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     last_result: serde_json::Value,
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
-    base_internal_url: &str,
     rsmq: Option<R>,
 ) -> anyhow::Result<()> {
     let value = flow_job
@@ -909,7 +899,6 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         last_result,
         same_worker_tx,
         worker_dir,
-        base_internal_url,
         rsmq,
     )
     .await?;
@@ -927,7 +916,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
     mut last_result: serde_json::Value,
     same_worker_tx: Sender<Uuid>,
     worker_dir: &str,
-    base_internal_url: &str,
     rsmq: Option<R>,
 ) -> error::Result<()> {
     let job_root = flow_job
@@ -964,7 +952,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             same_worker_tx,
             worker_dir,
             None,
-            base_internal_url,
             rsmq,
         )
         .await;
@@ -994,7 +981,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 same_worker_tx,
                 worker_dir,
                 Some(true),
-                base_internal_url,
                 rsmq,
             )
             .await;
@@ -1167,7 +1153,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 let logs = "Timed out waiting to be resumed".to_string();
                 let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}});
                 let _uuid =
-                    add_completed_job(db, &flow_job, success, skipped, result, logs, rsmq).await?;
+                    add_completed_job(db, &flow_job, success, skipped, &result, logs, rsmq).await?;
 
                 return Ok(());
             }
@@ -1303,7 +1289,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             FlowModuleValue::Script { input_transforms, .. }
             | FlowModuleValue::RawScript { input_transforms, .. }
             | FlowModuleValue::Flow { input_transforms, .. } => {
-                let ctx = get_transform_context(&flow_job, previous_id.clone(), &status).await?;
+                let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
                 transform_context = Some(ctx);
                 let by_id = transform_context.as_ref().unwrap();
                 transform_input(
@@ -1351,7 +1337,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         &status,
         &status_module,
         last_result.clone(),
-        previous_id,
+        &previous_id,
         client,
         resume_messages.as_slice(),
         approvers.clone(),
@@ -1402,29 +1388,10 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         let payload_tag = match &job_payloads {
             ContinuePayload::SingleJob(payload) => payload.clone(),
             ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
-            ContinuePayload::ForloopJobs { modules, .. } => {
-                let mut fm = flow.failure_module.clone();
-                if let Some(mut failure_module) = flow.failure_module.clone() {
-                    failure_module.id_append(&format!("{}/{}", status.step, i));
-                    fm = Some(failure_module);
-                }
-                JobPayloadWithTag {
-                    payload: JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules: (*modules).clone(),
-                            failure_module: fm.clone(),
-                            same_worker: flow.same_worker,
-                            concurrent_limit: None,
-                            concurrency_time_window_s: None,
-                            skip_expr: None,
-                            cache_ttl: None,
-                        },
-                        path: Some(format!("{}/forloop", flow_job.script_path())),
-                    },
-                    tag: None,
-                }
-            }
+            ContinuePayload::ForloopJobs { payload, .. } => payload.clone(),
         };
+
+        let transform_inp;
         let args = match &next_status {
             NextStatus::AllFlowJobs {
                 branchall: Some(BranchAllStatus { .. }),
@@ -1441,15 +1408,34 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             NextStatus::AllFlowJobs {
                 branchall: None,
                 iterator: Some(Iterator { itered, .. }),
-                ..
-            } => args.as_ref().map(|args| {
-                let mut new_args = args.clone();
-                new_args.insert(
-                    "iter".to_string(),
-                    json!({ "index": i, "value": itered[i] }),
-                );
-                new_args
-            }),
+                simple_input_transforms,
+            } => {
+                if let Ok(args) = args.as_ref() {
+                    let mut new_args = args.clone();
+                    new_args.insert(
+                        "iter".to_string(),
+                        json!({ "index": i, "value": itered[i] }),
+                    );
+                    if let Some(input_transforms) = simple_input_transforms {
+                        let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
+                        transform_inp = transform_input(
+                            &Some(serde_json::Value::Object(new_args)),
+                            last_result.clone(),
+                            input_transforms,
+                            resume_messages.as_slice(),
+                            approvers.clone(),
+                            &ctx,
+                            client,
+                        )
+                        .await;
+                        transform_inp.as_ref().map(|args| args.clone())
+                    } else {
+                        Ok(new_args)
+                    }
+                } else {
+                    args.as_ref().map(|args| args.clone())
+                }
+            }
             _ => args.as_ref().map(|args| args.clone()),
         };
         let (ok, err) = match args {
@@ -1537,7 +1523,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 parallel: false,
             }
         }
-        NextStatus::AllFlowJobs { iterator, branchall } => FlowStatusModule::InProgress {
+        NextStatus::AllFlowJobs { iterator, branchall, .. } => FlowStatusModule::InProgress {
             job: flow_job.id,
             iterator,
             flow_jobs: Some(uuids),
@@ -1722,6 +1708,7 @@ enum NextStatus {
     AllFlowJobs {
         branchall: Option<BranchAllStatus>,
         iterator: Option<windmill_common::flow_status::Iterator>,
+        simple_input_transforms: Option<HashMap<String, InputTransform>>,
     },
 }
 
@@ -1732,7 +1719,7 @@ struct JobPayloadWithTag {
 }
 enum ContinuePayload {
     SingleJob(JobPayloadWithTag),
-    ForloopJobs { n: usize, modules: Vec<FlowModule> },
+    ForloopJobs { n: usize, payload: JobPayloadWithTag },
     BranchAllJobs(Vec<JobPayloadWithTag>),
 }
 
@@ -1750,7 +1737,7 @@ async fn compute_next_flow_transform(
     status: &FlowStatus,
     status_module: &FlowStatusModule,
     last_result: serde_json::Value,
-    previous_id: String,
+    previous_id: &str,
     client: &AuthedClient,
     resumes: &[Value],
     approvers: Vec<String>,
@@ -1773,33 +1760,16 @@ async fn compute_next_flow_transform(
     match &module.value {
         FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
         FlowModuleValue::Flow { path, .. } => {
-            let payload = JobPayload::Flow(path.to_string());
+            let payload = flow_to_payload(path);
             Ok(NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag: None }),
+                ContinuePayload::SingleJob(payload),
                 NextStatus::NextStep,
             ))
         }
         FlowModuleValue::Script { path: script_path, hash: script_hash, .. } => {
-            let (payload, tag) = if script_hash.is_none() {
-                script_path_to_payload(script_path, &db, &flow_job.workspace_id).await?
-            } else {
-                let hash = script_hash.clone().unwrap();
-                let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = db.begin().await?;
-                let (tag, concurrent_limit, concurrency_time_window_s, cache_ttl) =
-                    script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?;
-                (
-                    JobPayload::ScriptHash {
-                        hash,
-                        path: script_path.to_owned(),
-                        concurrent_limit,
-                        concurrency_time_window_s,
-                        cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(),
-                    },
-                    tag,
-                )
-            };
+            let payload = script_to_payload(script_hash, script_path, db, flow_job, module).await?;
             Ok(NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag }),
+                ContinuePayload::SingleJob(payload),
                 NextStatus::NextStep,
             ))
         }
@@ -1816,25 +1786,27 @@ async fn compute_next_flow_transform(
             let path = path
                 .clone()
                 .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
+            let payload = raw_script_to_payload(
+                path,
+                content,
+                language,
+                lock,
+                concurrent_limit,
+                concurrency_time_window_s,
+                module,
+                tag,
+            );
             Ok(NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayloadWithTag {
-                    payload: JobPayload::Code(RawCode {
-                        path,
-                        content: content.clone(),
-                        language: language.clone(),
-                        lock: lock.clone(),
-                        concurrent_limit: *concurrent_limit,
-                        concurrency_time_window_s: *concurrency_time_window_s,
-                        cache_ttl: module.cache_ttl.map(|x| x as i32),
-                    }),
-                    tag: tag.clone(),
-                }),
+                ContinuePayload::SingleJob(payload),
                 NextStatus::NextStep,
             ))
         }
         /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
         FlowModuleValue::ForloopFlow { modules, iterator, parallel, .. } => {
             let new_args: &mut Map<String, serde_json::Value> = &mut Map::new();
+            // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
+            let is_simple =
+                modules.len() == 1 && modules[0].value.is_simple() && flow.failure_module.is_none();
 
             let next_loop_status = match status_module {
                 FlowStatusModule::WaitingForPriorSteps { .. }
@@ -1928,32 +1900,106 @@ async fn compute_next_flow_transform(
                         failure_module.id_append(&format!("{}/{}", status.step, ns.index));
                         fm = Some(failure_module);
                     }
-                    Ok(NextFlowTransform::Continue(
-                        ContinuePayload::SingleJob(JobPayloadWithTag {
-                            payload: JobPayload::RawFlow {
-                                value: FlowValue {
-                                    modules: (*modules).clone(),
-                                    failure_module: fm,
-                                    same_worker: flow.same_worker,
-                                    concurrent_limit: None,
-                                    concurrency_time_window_s: None,
-                                    skip_expr: None,
-                                    cache_ttl: None,
-                                },
-                                path: Some(format!("{}/loop-{}", flow_job.script_path(), ns.index)),
+                    let modules = (*modules).clone();
+                    let inner_path = Some(format!("{}/loop-{}", flow_job.script_path(), ns.index));
+                    let continue_payload = ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::RawFlow {
+                            value: FlowValue {
+                                modules,
+                                failure_module: fm,
+                                same_worker: flow.same_worker,
+                                concurrent_limit: None,
+                                concurrency_time_window_s: None,
+                                skip_expr: None,
+                                cache_ttl: None,
                             },
-                            tag: None,
-                        }),
+                            path: inner_path,
+                        },
+                        tag: None,
+                    });
+                    Ok(NextFlowTransform::Continue(
+                        continue_payload,
                         NextStatus::NextLoopIteration(ns),
                     ))
                 }
-                LoopStatus::ParallelIteration { itered, .. } => Ok(NextFlowTransform::Continue(
-                    ContinuePayload::ForloopJobs { n: itered.len(), modules: (*modules).clone() },
-                    NextStatus::AllFlowJobs {
-                        branchall: None,
-                        iterator: Some(windmill_common::flow_status::Iterator { index: 0, itered }),
-                    },
-                )),
+                LoopStatus::ParallelIteration { itered, .. } => {
+                    let inner_path = Some(format!("{}/loop-parrallel", flow_job.script_path(),));
+                    let continue_payload = if is_simple {
+                        let payload = match &modules[0].value {
+                            FlowModuleValue::Flow { path, .. } => flow_to_payload(path),
+                            FlowModuleValue::Script {
+                                path: script_path,
+                                hash: script_hash,
+                                ..
+                            } => {
+                                script_to_payload(script_hash, script_path, db, flow_job, module)
+                                    .await?
+                            }
+                            FlowModuleValue::RawScript {
+                                path,
+                                content,
+                                language,
+                                lock,
+                                tag,
+                                concurrent_limit,
+                                concurrency_time_window_s,
+                                ..
+                            } => raw_script_to_payload(
+                                path.clone().or(inner_path),
+                                content,
+                                language,
+                                lock,
+                                concurrent_limit,
+                                concurrency_time_window_s,
+                                module,
+                                tag,
+                            ),
+                            _ => unreachable!("is simple flow"),
+                        };
+                        ContinuePayload::ForloopJobs { n: itered.len(), payload: payload }
+                    } else {
+                        let payload = {
+                            JobPayloadWithTag {
+                                payload: JobPayload::RawFlow {
+                                    value: FlowValue {
+                                        modules: (*modules).clone(),
+                                        failure_module: flow.failure_module.clone(),
+                                        same_worker: flow.same_worker,
+                                        concurrent_limit: None,
+                                        concurrency_time_window_s: None,
+                                        skip_expr: None,
+                                        cache_ttl: None,
+                                    },
+                                    path: Some(format!("{}/forloop", flow_job.script_path())),
+                                },
+                                tag: None,
+                            }
+                        };
+                        ContinuePayload::ForloopJobs { n: itered.len(), payload }
+                    };
+                    Ok(NextFlowTransform::Continue(
+                        continue_payload,
+                        NextStatus::AllFlowJobs {
+                            branchall: None,
+                            iterator: Some(windmill_common::flow_status::Iterator {
+                                index: 0,
+                                itered,
+                            }),
+                            simple_input_transforms: if is_simple {
+                                match &modules[0].value {
+                                    FlowModuleValue::Script { input_transforms, .. }
+                                    | FlowModuleValue::RawScript { input_transforms, .. }
+                                    | FlowModuleValue::Flow { input_transforms, .. } => {
+                                        Some(input_transforms.clone())
+                                    }
+                                    _ => None,
+                                }
+                            } else {
+                                None
+                            },
+                        },
+                    ))
+                }
             }
         }
         FlowModuleValue::BranchOne { branches, default, .. } => {
@@ -2077,6 +2123,7 @@ async fn compute_next_flow_transform(
                                     len: branches.len(),
                                 }),
                                 iterator: None,
+                                simple_input_transforms: None,
                             },
                         ));
                     } else {
@@ -2148,9 +2195,74 @@ async fn compute_next_flow_transform(
     }
 }
 
+fn raw_script_to_payload(
+    path: Option<String>,
+    content: &String,
+    language: &windmill_common::scripts::ScriptLang,
+    lock: &Option<String>,
+    concurrent_limit: &Option<i32>,
+    concurrency_time_window_s: &Option<i32>,
+    module: &FlowModule,
+    tag: &Option<String>,
+) -> JobPayloadWithTag {
+    JobPayloadWithTag {
+        payload: JobPayload::Code(RawCode {
+            path,
+            content: content.clone(),
+            language: language.clone(),
+            lock: lock.clone(),
+            concurrent_limit: *concurrent_limit,
+            concurrency_time_window_s: *concurrency_time_window_s,
+            cache_ttl: module.cache_ttl.map(|x| x as i32),
+        }),
+        tag: tag.clone(),
+    }
+}
+
+fn flow_to_payload(path: &str) -> JobPayloadWithTag {
+    let payload = JobPayload::Flow(path.to_string());
+    JobPayloadWithTag { payload, tag: None }
+}
+
+async fn script_to_payload(
+    script_hash: &Option<windmill_common::scripts::ScriptHash>,
+    script_path: &String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    flow_job: &QueuedJob,
+    module: &FlowModule,
+) -> Result<JobPayloadWithTag, Error> {
+    let (payload, tag) = if script_hash.is_none() {
+        script_path_to_payload(script_path, &db, &flow_job.workspace_id).await?
+    } else {
+        let hash = script_hash.clone().unwrap();
+        let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = db.begin().await?;
+        let (
+            tag,
+            concurrent_limit,
+            concurrency_time_window_s,
+            cache_ttl,
+            language,
+            dedicated_worker,
+        ) = script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?;
+        (
+            JobPayload::ScriptHash {
+                hash,
+                path: script_path.to_owned(),
+                concurrent_limit,
+                concurrency_time_window_s,
+                cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(),
+                language,
+                dedicated_worker,
+            },
+            tag,
+        )
+    };
+    Ok(JobPayloadWithTag { payload, tag })
+}
+
 async fn get_transform_context(
     flow_job: &QueuedJob,
-    previous_id: String,
+    previous_id: &str,
     status: &FlowStatus,
 ) -> error::Result<IdContext> {
     let steps_results: HashMap<String, JobResult> = status
@@ -2159,7 +2271,7 @@ async fn get_transform_context(
         .filter_map(|x| x.job_result().map(|y| (x.id(), y)))
         .collect();
 
-    Ok(IdContext { flow_job: flow_job.id, steps_results, previous_id })
+    Ok(IdContext { flow_job: flow_job.id, steps_results, previous_id: previous_id.to_string() })
 }
 
 async fn evaluate_with<F>(

benchmarks/action.ts

@@ -2,7 +2,7 @@ import {
   FlowValue,
   JobService,
   Preview as ScriptPreview,
-} from "https://deno.land/x/windmill@v1.38.5/windmill-api/index.ts";
+} from "https://deno.land/x/windmill@v1.167.0/windmill-api/index.ts";
 
 export type Action = FlowAction | ScriptAction | RandomAction;
 

benchmarks/benchmark_noop.ts

@@ -20,199 +20,231 @@ async function login(email: string, password: string): Promise<string> {
   });
 }
 
-export const VERSION = "v1.125.1";
+export const VERSION = "v1.167.0";
 
-await new Command()
-  .name("wmillbench")
-  .description("Run Benchmark to measure throughput of windmill.")
-  .version(VERSION)
-  .option("--host <url:string>", "The windmill host to benchmark.", {
-    default: "http://127.0.0.1:8000",
-  })
-  .option("-e --email <email:string>", "The email to use to login.")
-  .option("-p --password <password:string>", "The password to use to login.")
-  .env(
-    "WM_TOKEN=<token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
-  )
-  .option(
-    "-t --token <token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
-  )
-  .env(
-    "WM_WORKSPACE=<workspace:string>",
-    "The workspace to spawn scripts from."
-  )
-  .option(
-    "-w --workspace <workspace:string>",
-    "The workspace to spawn scripts from.",
-    { default: "admins" }
-  )
-  .option(
-    "-j --jobs <jobs:number>",
-    "Number of NOOP jobs to create.",
-    { default: 10000 }
-  )
-  .option(
-    "-b --batches <batches:number>",
-    "Number of batches to create all the jobs.",
-    { default: 1 }
-  )
-  .action(
-    async ({
-      host,
-      email,
-      password,
-      token,
-      workspace,
-      jobs,
-      batches,
-    }) => {
-      windmill.setClient("", host);
+export async function main({
+  host,
+  email,
+  password,
+  token,
+  workspace,
+  jobs,
+  batches,
+}: {
+  host: string;
+  email?: string;
+  password?: string;
+  token?: string;
+  workspace: string;
+  jobs: number;
+  batches: number;
+}) {
+  windmill.setClient("", host);
 
-      console.log(
-        "Started benchmark with NOOP jobs with options",
-        JSON.stringify(
-          {
-            host,
-            email,
-            workspace,
-          },
-          null,
-          4
-        )
-      );
+  console.log(
+    "Started benchmark with NOOP jobs with options",
+    JSON.stringify(
+      {
+        host,
+        email,
+        workspace,
+      },
+      null,
+      4
+    )
+  );
 
-      const config = {
-        token: "",
-        server: host,
-        workspace_id: workspace,
-      };
+  const config = {
+    token: "",
+    server: host,
+    workspace_id: workspace,
+  };
 
-      let final_token: string;
-      if (!token) {
-        if (email && password) {
-          final_token = await login(email, password);
-        } else {
-          console.error("Token or email with password are required.");
-          return;
-        }
-      } else {
-        final_token = token;
-      }
-
-      config.token = final_token;
-      windmill.setClient(final_token, host);
-      const enc = (s: string) => new TextEncoder().encode(s);
-
-      console.log("Disabling workers before loading jobs")
-      const disable_workers = await fetch(
-        config.server + "/api/workers/toggle?disable=true",
-        {
-          method: "GET",
-          headers: { ["Authorization"]: "Bearer " + config.token },
-        }
-      )
-      if (!disable_workers.ok) {
-        console.error("Unable to disable workers. Is the Windmill server running in benchmark mode?")
-      }
-
-      const jobsSent = jobs;
-      const batch_num = batches;
-      console.log(`Bulk creating ${jobsSent} jobs in ${batch_num} batches`)
-
-      const start_create = Date.now()
-      const all_create_operations = []
-      for (let i = 0; i < batch_num; i++) {
-        all_create_operations.push(fetch(
-          config.server +
-            "/api/w/" +
-            config.workspace_id +
-            `/jobs/add_noop_jobs/${jobsSent / batch_num}`,
-          {
-            method: "POST",
-            headers: { ["Authorization"]: "Bearer " + config.token },
-          }
-        ));
-      }
-      await Promise.all(all_create_operations)
-
-      const end_create = Date.now()
-      const create_duration = end_create - start_create
-      console.log(`Jobs successfully added to the queue in ${create_duration}s. Windmill will start pulling them\n`)
-      const start = Date.now()
-
-      let queue_length = jobsSent
-      const updateState = setInterval(async () => {
-        const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
-        queue_length = (
-          await (
-            await fetch(
-              host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-              { headers: { ["Authorization"]: "Bearer " + config.token } }
-            )
-          ).json()
-        ).database_length;
-        await Deno.stdout.write(
-          enc(
-            `elapsed: ${elapsed} | jobs executed: ${JSON.stringify(
-              jobsSent - queue_length
-            )}/${jobsSent} (thr: ${((jobsSent - queue_length) / elapsed).toFixed(
-              2
-            )}) | queue: ${queue_length}                          \r`
-          )
-        );
-      }, 100);
-
-      console.log("Enabling workers to start processing jobs")
-      const enable_workers = await fetch(
-        config.server + "/api/workers/toggle?disable=false",
-        {
-          method: "GET",
-          headers: { ["Authorization"]: "Bearer " + config.token },
-        }
-      )
-      if (!enable_workers.ok) {
-        console.error("Unable to disable workers. Is the Windmill server running in benchmark mode?")
-      }
-
-      while (queue_length > 0) {
-        await sleep(0.1);
-      }
-
-      clearInterval(updateState);
-
-      const total_duration_sec = (Date.now() - start) / 1000;
-      console.log(`jobs: ${jobsSent}`);
-      console.log(`duration: ${total_duration_sec}s`);
-      console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`);
-
-      console.log(
-        "queue length:",
-        (
-          await (
-            await fetch(
-              host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-              { headers: { ["Authorization"]: "Bearer " + config.token } }
-            )
-          ).json()
-        ).database_length
-      );
-      console.log("done");
+  let final_token: string;
+  if (!token) {
+    if (email && password) {
+      final_token = await login(email, password);
+    } else {
+      console.error("Token or email with password are required.");
+      return;
     }
-  )
-  .command(
-    "upgrade",
-    new UpgradeCommand({
-      main: "main.ts",
-      args: [
-        "--allow-net",
-        "--allow-read",
-        "--allow-write",
-        "--allow-env",
-        "--unstable",
-      ],
-      provider: new DenoLandProvider({ name: "wmillbench" }),
+  } else {
+    final_token = token;
+  }
+
+  config.token = final_token;
+  windmill.setClient(final_token, host);
+  const enc = (s: string) => new TextEncoder().encode(s);
+
+  console.log("Disabling workers before loading jobs");
+  const disable_workers = await fetch(
+    config.server + "/api/workers/toggle?disable=true",
+    {
+      method: "GET",
+      headers: { ["Authorization"]: "Bearer " + config.token },
+    }
+  );
+  if (!disable_workers.ok) {
+    console.error(
+      "Unable to disable workers. Is the Windmill server running in benchmark mode?"
+    );
+  }
+
+  const jobsSent = jobs;
+  const batch_num = batches;
+  console.log(`Bulk creating ${jobsSent} jobs in ${batch_num} batches`);
+
+  const start_create = Date.now();
+  const all_create_operations = [];
+  for (let i = 0; i < batch_num; i++) {
+    all_create_operations.push(
+      fetch(
+        config.server +
+          "/api/w/" +
+          config.workspace_id +
+          `/jobs/add_noop_jobs/${jobsSent / batch_num}`,
+        {
+          method: "POST",
+          headers: { ["Authorization"]: "Bearer " + config.token },
+        }
+      )
+    );
+  }
+  await Promise.all(all_create_operations);
+
+  const end_create = Date.now();
+  const create_duration = end_create - start_create;
+  console.log(
+    `Jobs successfully added to the queue in ${create_duration}s. Windmill will start pulling them\n`
+  );
+  const start = Date.now();
+
+  let queue_length = jobsSent;
+  let lastElapsed = 0;
+  let lastQueueLength = queue_length;
+  const updateState = setInterval(async () => {
+    const elapsed = start ? Date.now() - start : 0;
+    queue_length = (
+      await (
+        await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length;
+    const avgThr = (((jobsSent - queue_length) / elapsed) * 1000).toFixed(2);
+    const instThr =
+      lastElapsed > 0
+        ? (
+            ((lastQueueLength - queue_length) / (elapsed - lastElapsed)) *
+            1000
+          ).toFixed(2)
+        : 0;
+
+    lastElapsed = elapsed;
+    lastQueueLength = queue_length;
+
+    await Deno.stdout.write(
+      enc(
+        `elapsed: ${(elapsed / 1000).toFixed(2)} | jobs executed: ${
+          jobsSent - queue_length
+        }/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | queue: ${queue_length}                          \r`
+      )
+    );
+  }, 100);
+
+  console.log("Enabling workers to start processing jobs");
+  const enable_workers = await fetch(
+    config.server + "/api/workers/toggle?disable=false",
+    {
+      method: "GET",
+      headers: { ["Authorization"]: "Bearer " + config.token },
+    }
+  );
+  if (!enable_workers.ok) {
+    console.error(
+      "Unable to disable workers. Is the Windmill server running in benchmark mode?"
+    );
+  }
+
+  while (queue_length > 0) {
+    await sleep(0.1);
+  }
+
+  clearInterval(updateState);
+
+  const total_duration_sec = (Date.now() - start) / 1000.0;
+  console.log(`jobs: ${jobsSent}`);
+  console.log(`duration: ${total_duration_sec}s`);
+  console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`);
+
+  console.log(
+    "queue length:",
+    (
+      await (
+        await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length
+  );
+  console.log("done");
+
+  return {
+    throughput: jobsSent / total_duration_sec,
+  };
+}
+
+if (import.meta.main) {
+  await new Command()
+    .name("wmillbench")
+    .description("Run Benchmark to measure throughput of windmill.")
+    .version(VERSION)
+    .option("--host <url:string>", "The windmill host to benchmark.", {
+      default: "http://127.0.0.1:8000",
     })
-  )
-  .parse();
+    .option("-e --email <email:string>", "The email to use to login.")
+    .option("-p --password <password:string>", "The password to use to login.")
+    .env(
+      "WM_TOKEN=<token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .option(
+      "-t --token <token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .env(
+      "WM_WORKSPACE=<workspace:string>",
+      "The workspace to spawn scripts from."
+    )
+    .option(
+      "-w --workspace <workspace:string>",
+      "The workspace to spawn scripts from.",
+      { default: "admins" }
+    )
+    .option("-j --jobs <jobs:number>", "Number of NOOP jobs to create.", {
+      default: 10000,
+    })
+    .option(
+      "-b --batches <batches:number>",
+      "Number of batches to create all the jobs.",
+      { default: 1 }
+    )
+    .action(main)
+    .command(
+      "upgrade",
+      new UpgradeCommand({
+        main: "main.ts",
+        args: [
+          "--allow-net",
+          "--allow-read",
+          "--allow-write",
+          "--allow-env",
+          "--unstable",
+        ],
+        provider: new DenoLandProvider({ name: "wmillbench" }),
+      })
+    )
+    .parse();
+}

benchmarks/benchmark_suite.ts

@@ -0,0 +1,194 @@
+import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
+import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
+import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
+
+const VERSION = "1.167.0";
+
+type Config = {
+  benchmarks: [
+    {
+      graph_title: string;
+      name: string;
+      jobs: number | undefined;
+      type: "noop" | "flow" | "deno" | "python" | "go" | "bash";
+    }
+  ];
+};
+
+async function main({
+  host,
+  email,
+  password,
+  token,
+  workspace,
+  configPath,
+  branch,
+}: {
+  host: string;
+  email?: string;
+  password?: string;
+  token?: string;
+  workspace: string;
+  configPath: string;
+  branch?: string;
+}) {
+  const { main: runNoopBenchmark } = await import(
+    branch !== undefined
+      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/benchmark_noop.ts`
+      : "./benchmark_noop.ts"
+  );
+
+  const { main: runBenchmark } = await import(
+    branch !== undefined
+      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/main.ts`
+      : "./main.ts"
+  );
+
+  const { drawGraph } = await import(
+    branch !== undefined
+      ? `https://raw.githubusercontent.com/windmill-labs/windmill/${branch}/benchmarks/graph.ts`
+      : "./graph.ts"
+  );
+
+  async function getConfig(configPath: string): Promise<Config> {
+    if (configPath.startsWith("http")) {
+      const response = await fetch(configPath);
+      return await response.json();
+    } else {
+      return JSON.parse(await Deno.readTextFile(configPath));
+    }
+  }
+
+  try {
+    const config = await getConfig(configPath);
+    for (const benchmark of config.benchmarks) {
+      try {
+        console.log(
+          "%cRunning benchmark " + benchmark.name,
+          "font-weight: bold;"
+        );
+
+        let result:
+          | {
+              throughput: number;
+            }
+          | undefined;
+        if (benchmark.type === "noop") {
+          result = await runNoopBenchmark({
+            host,
+            email,
+            password,
+            token,
+            workspace,
+            jobs: 1000,
+            batches: 1,
+          });
+        } else {
+          result = await runBenchmark({
+            host,
+            email,
+            password,
+            token,
+            workspace,
+            workers: 1,
+            seconds: benchmark.type === "flow" ? 2 : 5,
+            metrics: "http://localhost:8001/metrics",
+            maximumThroughput: Infinity,
+            zombieTimeout: 90000,
+            histogramBuckets: [],
+            scriptPattern: [
+              "deno",
+              "python",
+              "go",
+              "bash",
+              "dedicated",
+            ].includes(benchmark.type)
+              ? benchmark.type
+              : "deno",
+            useFlows: benchmark.type === "flow",
+            hideProgress: true,
+          });
+        }
+
+        if (!result) {
+          throw new Error("No result returned");
+        }
+        const stat = {
+          value: result.throughput,
+          ts: Date.now(),
+        };
+        let data: (typeof stat)[] = [];
+        const jsonFilePath = `${benchmark.name}.json`;
+        try {
+          const existing = await Deno.readTextFile(jsonFilePath);
+          data = JSON.parse(existing);
+        } catch (_) {
+          console.log("No existing data file found, creating new one.");
+        }
+        data.push(stat);
+        await Deno.writeTextFile(jsonFilePath, JSON.stringify(data, null, 4));
+        const svg = drawGraph(
+          data.slice(-10).map((d) => ({ ...d, date: new Date(d.ts) })),
+          benchmark.graph_title
+        );
+        await Deno.writeTextFile(`${benchmark.name}.svg`, svg);
+      } catch (err) {
+        console.error("Failed to run benchmark", benchmark.name, err);
+      }
+    }
+
+    Deno.exit(0); // JSDOM from drawGraph doesn't exit cleanly
+  } catch (err) {
+    return console.error(`Failed to read config file ${configPath}: ${err}`);
+  }
+}
+
+await new Command()
+  .name("wmillbenchsuite")
+  .description("Run benchmark suite to measure throughput of windmill.")
+  .version(VERSION)
+  .option("--host <url:string>", "The windmill host to benchmark.", {
+    default: "http://127.0.0.1:8000",
+  })
+  .option("-e --email <email:string>", "The email to use to login.")
+  .option("-p --password <password:string>", "The password to use to login.")
+  .env(
+    "WM_TOKEN=<token:string>",
+    "The token to use when talking to the API server. Preferred over manual login."
+  )
+  .option(
+    "-t --token <token:string>",
+    "The token to use when talking to the API server. Preferred over manual login."
+  )
+  .env(
+    "WM_WORKSPACE=<workspace:string>",
+    "The workspace to spawn scripts from."
+  )
+  .option(
+    "-w --workspace <workspace:string>",
+    "The workspace to spawn scripts from.",
+    { default: "admins" }
+  )
+  .option("-c --config-path <config:string>", "The path of the config file", {
+    required: true,
+  })
+  .option(
+    "--branch <branch:string>",
+    "The branch to use when running remotely."
+  )
+  .action(main)
+  .command(
+    "upgrade",
+    new UpgradeCommand({
+      main: "main.ts",
+      args: [
+        "--allow-net",
+        "--allow-read",
+        "--allow-write",
+        "--allow-env",
+        "--unstable",
+      ],
+      provider: new DenoLandProvider({ name: "wmillbench" }),
+    })
+  )
+  .parse();

benchmarks/graph.ts

@@ -0,0 +1,127 @@
+import * as d3 from "https://cdn.jsdelivr.net/npm/d3@7/+esm";
+import { JSDOM } from "https://jspm.dev/jsdom";
+
+type DataPoint = {
+  value: number;
+  date: Date;
+};
+export function drawGraph(data: DataPoint[], title: string) {
+  const context = {
+    jsdom: new JSDOM(""),
+  };
+  const { window } = context.jsdom;
+  const { document } = window;
+
+  const body = d3.select(document).select("body");
+
+  const width = 400;
+  const height = 200;
+
+  const marginTop = 20;
+  const marginRight = 30;
+  const marginBottom = 30;
+  const marginLeft = 60;
+
+  let svg = body
+    .append("svg")
+    .attr("xmlns", "http://www.w3.org/2000/svg")
+    .attr("width", width + marginLeft + marginRight)
+    .attr("height", height + marginTop + marginBottom);
+
+  svg
+    .append("rect")
+    .attr("width", "100%")
+    .attr("height", "100%")
+    .attr("fill", "white");
+
+  svg = svg
+    .append("g")
+    .attr("transform", "translate(" + marginLeft + "," + marginTop + ")");
+
+  const x = d3
+    .scaleTime()
+    .domain(
+      d3.extent(data, function (d: DataPoint) {
+        return d.date;
+      })
+    )
+    .nice()
+    .range([0, width]);
+
+  const xAxis = d3.axisBottom(x).ticks(5);
+
+  svg
+    .append("g")
+    .attr("transform", "translate(0," + height + ")")
+    .call(xAxis);
+
+  // Add Y axis
+  const y = d3
+    .scaleLinear()
+    .domain([
+      0,
+      d3.max(data, function (d: DataPoint) {
+        return +d.value;
+      }) * 1.5,
+    ])
+    .range([height, 0])
+    .nice();
+  svg.append("g").call(d3.axisLeft(y));
+
+  svg
+    .append("text")
+    .attr("text-anchor", "middle")
+    .attr("style", "font-size: 12px")
+    .attr("transform", "rotate(-90)")
+    .attr("y", -marginLeft + 20)
+    .attr("x", -height / 2)
+    .text("[jobs/s]");
+
+  svg
+    .append("text")
+    .attr("text-anchor", "middle")
+    .attr("style", "font-size: 16px")
+    .attr("y", 0)
+    .attr("x", width / 2)
+    .text(title);
+
+  // Add the line
+  svg
+    .append("path")
+    .datum(data)
+    .attr("fill", "none")
+    .attr("stroke", "steelblue")
+    .attr("stroke-width", 1.5)
+    .attr(
+      "d",
+      d3.line(
+        function (d: DataPoint) {
+          return x(d.date);
+        },
+        function (d: DataPoint) {
+          return y(d.value);
+        }
+      )
+    );
+
+  return body.node().innerHTML;
+}
+
+if (import.meta.main) {
+  const svg = drawGraph(
+    [
+      {
+        value: 10,
+        date: new Date(86400000),
+      },
+      {
+        value: 12,
+        date: new Date(86400000 * 2),
+      },
+    ],
+    "test"
+  );
+
+  console.log(svg);
+  Deno.exit(0);
+}

benchmarks/main.ts

@@ -3,7 +3,8 @@
 
 import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
 import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
-import * as windmill from "https://deno.land/x/windmill@v1.38.5/mod.ts";
+import * as windmill from "https://deno.land/x/windmill@v1.167.0/mod.ts";
+import * as api from "https://deno.land/x/windmill@v1.167.0/windmill-api/index.ts";
 import { Action } from "./action.ts";
 import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
 import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
@@ -21,426 +22,519 @@ async function login(email: string, password: string): Promise<string> {
   });
 }
 
-export const VERSION = "v1.162.0";
+export const VERSION = "v1.168.3";
 
-await new Command()
-  .name("wmillbench")
-  .description("Run Benchmark to measure throughput of windmill.")
-  .version(VERSION)
-  .option("--host <url:string>", "The windmill host to benchmark.", {
-    default: "http://127.0.0.1:8000",
-  })
-  .option(
-    "--workers <workers:number>",
-    "The number of workers to run at once.",
-    {
-      default: 1,
-    }
-  )
-  .option(
-    "-s --seconds <seconds:number>",
-    "How long to run the benchmark for (in seconds).",
-    {
-      default: 30,
-    }
-  )
-  .option("--max <max:number>", "Maximum number of operations performed.")
-  .option("-e --email <email:string>", "The email to use to login.")
-  .option("-p --password <password:string>", "The password to use to login.")
-  .env(
-    "WM_TOKEN=<token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
-  )
-  .option(
-    "-t --token <token:string>",
-    "The token to use when talking to the API server. Preferred over manual login."
-  )
-  .env(
-    "WM_WORKSPACE=<workspace:string>",
-    "The workspace to spawn scripts from."
-  )
-  .option(
-    "-w --workspace <workspace:string>",
-    "The workspace to spawn scripts from.",
-    { default: "admins" }
-  )
-  .option("-m --metrics <metrics:string>", "The url to scrape metrics from.", {
-    default: "http://localhost:8001/metrics",
-  })
-  .option(
-    "--export-json <export_json:string>",
-    "If set, exports will be into a JSON file."
-  )
-  .option(
-    "--export-csv <export_csv:string>",
-    "If set, exports will be into a csv file."
-  )
-  .option(
-    "--export-histograms [histograms...:string]",
-    "Mark metrics (without label) that are reported as histograms to export."
-  )
-  .option(
-    "--export-simple [simple...:string]",
-    "Mark metrics (without label) that are reported as simple values."
-  )
-  .option(
-    "--maximum-throughput <maximum_throughput:number>",
-    "Maximum number of jobs/flows to start in one second.",
-    {
-      default: Infinity,
-    }
-  )
-  .option("--use-flows", "Run flows instead of jobs.")
-  .option(
-    "--flow-pattern <pattern:string>",
-    "Use a different flow pattern among: 2steps, onebranch (Default 2steps)"
-  )
-  .option(
-    "--script-pattern <pattern:string>",
-    "Use a different script pattern among: denotrivial, identity, httpversion, httpslow (Default denotrivial)"
-  )
-  .option("--custom <custom_path:string>", "Use custom actions during bench")
-  .option(
-    "--zombie-timeout",
-    "The maximum time in ms to wait for jobs to complete.",
-    {
-      default: 90000,
-    }
-  )
-  .option(
-    "--continous",
-    "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked."
-  )
-  .option(
-    "--histogram-buckets [buckets...:string]",
-    "Define what buckets to collect from histograms.",
-    {
-      default: [
-        "+Inf",
-        "10",
-        "5",
-        "2.5",
-        "2.5",
-        "1",
-        "0.5",
-        "0.25",
-        "0.1",
-        "0.05",
-        "0.025",
-        "0.01",
-        "0.005",
-      ],
-    }
-  )
-  .action(
-    async ({
-      host,
-      workers: num_workers,
-      seconds,
-      email,
-      password,
-      token,
-      workspace,
-      metrics,
-      exportJson,
-      exportCsv,
-      exportHistograms,
-      exportSimple,
-      histogramBuckets,
-      maximumThroughput,
-      useFlows,
-      flowPattern,
-      scriptPattern,
-      zombieTimeout,
-      continous,
-      max,
-      custom,
-    }) => {
-      windmill.setClient("", host);
+export async function main({
+  host,
+  workers: num_workers,
+  seconds,
+  email,
+  password,
+  token,
+  workspace,
+  metrics,
+  exportJson,
+  exportCsv,
+  exportHistograms,
+  exportSimple,
+  histogramBuckets,
+  maximumThroughput,
+  useFlows,
+  flowPattern,
+  scriptPattern,
+  zombieTimeout,
+  continous,
+  max,
+  custom,
+  hideProgress,
+}: {
+  host: string;
+  workers: number;
+  seconds: number;
+  email?: string;
+  password?: string;
+  token?: string;
+  workspace: string;
+  metrics: string;
+  exportJson?: string;
+  exportCsv?: string;
+  exportHistograms?: string[];
+  exportSimple?: string[];
+  histogramBuckets: string[];
+  maximumThroughput: number;
+  useFlows?: boolean;
+  flowPattern?: string;
+  scriptPattern?: string;
+  zombieTimeout: number;
+  continous?: boolean;
+  max?: number;
+  custom?: string;
+  hideProgress?: boolean;
+}) {
+  windmill.setClient("", host);
+  const versionResp = await fetch(`${host}/api/version`);
+  console.log("Backend version: " + (await versionResp.text()));
 
-      const custom_content: Action | undefined = custom
-        ? JSON.parse(await Deno.readTextFile(custom))
-        : undefined;
+  const custom_content: Action | undefined = custom
+    ? JSON.parse(await Deno.readTextFile(custom))
+    : undefined;
 
-      if (!Array.isArray(histogramBuckets)) {
-        histogramBuckets = [];
-      }
+  if (!Array.isArray(histogramBuckets)) {
+    histogramBuckets = [];
+  }
 
-      if (!Array.isArray(exportHistograms)) {
-        exportHistograms = [];
-      }
+  if (!Array.isArray(exportHistograms)) {
+    exportHistograms = [];
+  }
 
-      if (!Array.isArray(exportSimple)) {
-        exportSimple = [];
-      }
+  if (!Array.isArray(exportSimple)) {
+    exportSimple = [];
+  }
 
-      let metrics_worker: Worker | undefined = undefined;
-      if (!continous) {
-        if (exportJson || exportCsv) {
-          metrics_worker = new Worker(
-            new URL("./scraper.ts", import.meta.url).href,
-            {
-              type: "module",
-            }
-          );
-
-          metrics_worker.postMessage({
-            exportHistograms,
-            histogramBuckets,
-            exportSimple,
-            host: metrics,
-          });
+  let metrics_worker: Worker | undefined = undefined;
+  if (!continous) {
+    if (exportJson || exportCsv) {
+      metrics_worker = new Worker(
+        new URL("./scraper.ts", import.meta.url).href,
+        {
+          type: "module",
         }
-      }
-
-      console.log(
-        "Started with options",
-        JSON.stringify(
-          {
-            host,
-            num_workers,
-            seconds,
-            email,
-            workspace,
-            metrics,
-            exportJson,
-            exportCsv,
-            exportHistograms,
-            exportSimple,
-            maximumThroughput,
-            useFlows,
-            flowPattern,
-            scriptPattern,
-            zombieTimeout,
-          },
-          null,
-          4
-        )
       );
 
-      const config = {
-        token: "",
-        server: host,
-        workspace_id: workspace,
-      };
+      metrics_worker.postMessage({
+        exportHistograms,
+        histogramBuckets,
+        exportSimple,
+        host: metrics,
+      });
+    }
+  }
 
-      let final_token: string;
-      if (!token) {
-        if (email && password) {
-          final_token = await login(email, password);
-        } else {
-          console.error("Token or email with password are required.");
-          return;
-        }
-      } else {
-        final_token = token;
-      }
-
-      config.token = final_token;
-      windmill.setClient(final_token, host);
-
-      const per_worker_throughput = maximumThroughput / num_workers;
-      const max_per_worker = max ? max / num_workers : undefined;
-      const shared_config = {
-        server: host,
-        token: final_token,
-        workspace_id: config.workspace_id,
-        per_worker_throughput,
-        max_per_worker,
+  console.log(
+    "Started with options",
+    JSON.stringify(
+      {
+        host,
+        num_workers,
+        seconds,
+        email,
+        workspace,
+        metrics,
+        exportJson,
+        exportCsv,
+        exportHistograms,
+        exportSimple,
+        maximumThroughput,
         useFlows,
         flowPattern,
         scriptPattern,
+        zombieTimeout,
         continous,
-        custom: custom_content,
-      };
+        hideProgress,
+      },
+      null,
+      4
+    )
+  );
 
-      let workers: Worker[] = new Array(num_workers);
-      for (let i = 0; i < num_workers; i++) {
-        workers[i] = new Worker(new URL("./worker.ts", import.meta.url).href, {
-          type: "module",
-        });
-      }
+  const config = {
+    token: "",
+    server: host,
+    workspace_id: workspace,
+  };
 
-      let start: number | undefined = undefined;
+  let final_token: string;
+  if (!token) {
+    if (email && password) {
+      console.log("Logging in with email and password...");
+      final_token = await login(email, password);
+      console.log("Logged in!");
+    } else {
+      console.error("Token or email with password are required.");
+      return;
+    }
+  } else {
+    final_token = token;
+  }
 
-      const jobsSent = Array(num_workers).fill(0);
-      const enc = (s: string) => new TextEncoder().encode(s);
+  console.log("Using token", final_token);
 
-      const updateState = setInterval(async () => {
-        const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
-        const sum = jobsSent.reduce((a, b) => a + b, 0);
-        let queue_length = -1;
-        while (queue_length === -1) {
-          try {
-            queue_length = (
-              await (
-                await fetch(
-                  host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-                  { headers: { ["Authorization"]: "Bearer " + config.token } }
-                )
-              ).json()
-            ).database_length;
-          } catch (e) {
-            console.log(
-              `queue count not reachable. waiting...                                                           `
-            );
-            await sleep(0.5);
-            continue;
-          }
-        }
-        await Deno.stdout.write(
-          enc(
-            `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
-              jobsSent
-            )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
-              2
-            )}) | queue: ${queue_length}                          \r`
-          )
-        );
-      }, 100);
+  config.token = final_token;
+  windmill.setClient(final_token, host);
 
-      workers.forEach((worker, i) => {
-        worker.addEventListener("message", (evt: MessageEvent<any>) => {
-          if (evt.data.type === "jobs_sent") {
-            jobsSent[i] = evt.data.jobs_sent;
-          }
-        });
-        worker.postMessage({ ...shared_config, i });
+  const per_worker_throughput = maximumThroughput / num_workers;
+  const max_per_worker = max ? max / num_workers : undefined;
+  const shared_config = {
+    server: host,
+    token: final_token,
+    workspace_id: config.workspace_id,
+    per_worker_throughput,
+    max_per_worker,
+    useFlows,
+    flowPattern,
+    scriptPattern,
+    continous,
+    custom: custom_content,
+    hideProgress,
+  };
+
+  if (
+    !useFlows &&
+    (scriptPattern === undefined ||
+      ["deno", "python", "go", "bash"].includes(scriptPattern))
+  ) {
+    console.log("Creating benchmark script...");
+    const path = `f/benchmarks/${scriptPattern || "deno"}`;
+    const exists = await windmill.ScriptService.existsScriptByPath({
+      workspace,
+      path,
+    });
+
+    if (exists) {
+      await windmill.ScriptService.deleteScriptByPath({
+        workspace,
+        path,
       });
-      start = Date.now();
+    }
 
-      console.log("collecting samples...");
-      if (continous) {
-        while (true) {
-          await sleep(Infinity);
-        }
-      }
+    let scriptContent: string;
+    let language: string;
+    if (scriptPattern === "python") {
+      scriptContent =
+        'import os\n\ndef main():\n    return os.environ.get("WM_JOB_ID")';
+      language = "python3";
+    } else if (scriptPattern === "go") {
+      scriptContent =
+        'package inner\nimport "os"\nfunc main() (string, error) { return os.Getenv("WM_JOB_ID"), nil }';
+      language = "go";
+    } else if (scriptPattern === "bash") {
+      scriptContent = "echo $WM_JOB_ID";
+      language = "bash";
+    } else {
+      scriptContent =
+        'export function main(){ return Deno.env.get("WM_JOB_ID"); }';
+      language = "deno";
+    }
 
-      await sleep(seconds);
+    await windmill.ScriptService.createScript({
+      workspace,
+      requestBody: {
+        path,
+        content: scriptContent,
+        summary: (scriptPattern || "deno") + " benchmark",
+        description: "",
+        language: language as api.NewScript.language,
+      },
+    });
 
-      clearInterval(updateState);
+    await sleep(5); // make sure script is created
+  }
 
-      let sum = jobsSent.reduce((a, b) => a + b, 0);
-      await Deno.stdout.write(
-        enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
-      );
+  let workers: Worker[] = new Array(num_workers);
+  for (let i = 0; i < num_workers; i++) {
+    workers[i] = new Worker(new URL("./worker.ts", import.meta.url).href, {
+      type: "module",
+    });
+  }
 
-      const shutdown_start = Date.now();
-      let zombie_jobs = 0;
-      let incorrect_results = 0;
-      workers.forEach((worker, i) => {
-        const l = (evt: MessageEvent<any>) => {
-          if (evt.data.type === "zombie_jobs") {
-            zombie_jobs += evt.data.zombie_jobs;
-            incorrect_results += evt.data.incorrect_results;
-            worker.removeEventListener("message", l);
-            workers = workers.filter((w) => w != worker);
-            jobsSent[i] = evt.data.jobs_sent;
-            worker.terminate();
-          }
-        };
-        worker.addEventListener("message", l);
-        worker.postMessage(
-          Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
-        );
-      });
+  let start: number | undefined = undefined;
 
-      console.log("waiting for shutdown\n");
-      while (workers.length > 0) {
-        await sleep(0.1);
-      }
-      sum = jobsSent.reduce((a, b) => a + b, 0);
+  const jobsSent = Array(num_workers).fill(0);
+  const enc = (s: string) => new TextEncoder().encode(s);
 
-      const tts = (Date.now() - shutdown_start) / 1000;
-      const time = seconds + tts;
-      console.log("\ntime to shutdown:", tts);
-      console.log("jobs:", sum);
-      console.log("time (s + tts):", time);
-      console.log("throughput /s (jobs/time):", sum / time);
-
-      console.log("zombie jobs: ", zombie_jobs);
-      console.log("incorrect results: ", incorrect_results);
-      console.log(
-        "queue length:",
-        (
+  const updateState = setInterval(async () => {
+    const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
+    const sum = jobsSent.reduce((a, b) => a + b, 0);
+    let queue_length = -1;
+    while (queue_length === -1) {
+      try {
+        queue_length = (
           await (
             await fetch(
               host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
               { headers: { ["Authorization"]: "Bearer " + config.token } }
             )
           ).json()
-        ).database_length
-      );
-
-      if (metrics_worker) {
-        metrics_worker.postMessage("stop");
-        console.log("waiting for metrics");
-        const { columns, transfer_values } = await new Promise<{
-          columns: string[];
-          transfer_values: ArrayBufferLike[];
-        }>((resolve, _reject) => {
-          if (metrics_worker) {
-            metrics_worker.onmessage = (e) => {
-              resolve(e.data);
-              metrics_worker?.terminate();
-            };
-          }
-        });
-        const values = transfer_values.map((x) => new Float32Array(x));
-
-        if (exportJson) {
-          console.log("exporting mean & stdev to json");
-          const obj: any = {};
-          for (let i = 0; i < columns.length; i++) {
-            const name = columns[i]!;
-            const value = values[i]!;
-            const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
-            const stdev = Math.sqrt(
-              value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
-            );
-            obj[name] = { mean, stdev };
-          }
-
-          await Deno.writeTextFile(exportJson, JSON.stringify(obj));
-        }
-
-        if (exportCsv) {
-          const f = await Deno.open(exportCsv, {
-            write: true,
-            create: true,
-            truncate: true,
-          });
-          const encoder = new TextEncoder();
-          const newline = new Uint8Array(1);
-          newline[0] = 0x0a;
-          await f.write(encoder.encode(columns.join(",")));
-          await f.write(newline);
-
-          for (let i = 0; i < values.length; i++) {
-            await f.write(encoder.encode(values[i].join(",")));
-            await f.write(newline);
-          }
-
-          f.close();
-        }
-      } else {
-        return;
+        ).database_length;
+      } catch (e) {
+        console.log(
+          `queue count not reachable. waiting...                                                           `
+        );
+        await sleep(0.5);
+        continue;
       }
-      console.log("done");
     }
-  )
-  .command(
-    "upgrade",
-    new UpgradeCommand({
-      main: "main.ts",
-      args: [
-        "--allow-net",
-        "--allow-read",
-        "--allow-write",
-        "--allow-env",
-        "--unstable",
-      ],
-      provider: new DenoLandProvider({ name: "wmillbench" }),
+    await Deno.stdout.write(
+      enc(
+        `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
+          jobsSent
+        )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
+          2
+        )}) | queue: ${queue_length}                          \r`
+      )
+    );
+  }, 100);
+
+  workers.forEach((worker, i) => {
+    worker.addEventListener("message", (evt: MessageEvent<any>) => {
+      if (evt.data.type === "jobs_sent") {
+        jobsSent[i] = evt.data.jobs_sent;
+      }
+    });
+    worker.postMessage({ ...shared_config, i });
+  });
+  start = Date.now();
+
+  console.log("collecting samples...");
+  if (continous) {
+    while (true) {
+      await sleep(Infinity);
+    }
+  }
+
+  await sleep(seconds);
+
+  clearInterval(updateState);
+
+  let sum = jobsSent.reduce((a, b) => a + b, 0);
+  await Deno.stdout.write(
+    enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
+  );
+
+  const shutdown_start = Date.now();
+  let zombie_jobs = 0;
+  let incorrect_results = 0;
+  workers.forEach((worker, i) => {
+    const l = (evt: MessageEvent<any>) => {
+      if (evt.data.type === "zombie_jobs") {
+        zombie_jobs += evt.data.zombie_jobs;
+        incorrect_results += evt.data.incorrect_results;
+        worker.removeEventListener("message", l);
+        workers = workers.filter((w) => w != worker);
+        jobsSent[i] = evt.data.jobs_sent;
+        worker.terminate();
+      }
+    };
+    worker.addEventListener("message", l);
+    worker.postMessage(
+      Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
+    );
+  });
+
+  console.log("waiting for shutdown\n");
+  while (workers.length > 0) {
+    await sleep(0.1);
+  }
+  sum = jobsSent.reduce((a, b) => a + b, 0);
+
+  const tts = (Date.now() - shutdown_start) / 1000;
+  const time = seconds + tts;
+  console.log("\ntime to shutdown:", tts);
+  console.log("jobs:", sum);
+  console.log("time (s + tts):", time);
+  console.log("throughput /s (jobs/time):", sum / time);
+
+  console.log("zombie jobs: ", zombie_jobs);
+  console.log("incorrect results: ", incorrect_results);
+  console.log(
+    "queue length:",
+    (
+      await (
+        await fetch(
+          host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length
+  );
+
+  if (metrics_worker) {
+    metrics_worker.postMessage("stop");
+    console.log("waiting for metrics");
+    const { columns, transfer_values } = await new Promise<{
+      columns: string[];
+      transfer_values: ArrayBufferLike[];
+    }>((resolve, _reject) => {
+      if (metrics_worker) {
+        metrics_worker.onmessage = (e) => {
+          resolve(e.data);
+          metrics_worker?.terminate();
+        };
+      }
+    });
+    const values = transfer_values.map((x) => new Float32Array(x));
+
+    if (exportJson) {
+      console.log("exporting mean & stdev to json");
+      const obj: any = {};
+      for (let i = 0; i < columns.length; i++) {
+        const name = columns[i]!;
+        const value = values[i]!;
+        const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
+        const stdev = Math.sqrt(
+          value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
+        );
+        obj[name] = { mean, stdev };
+      }
+
+      await Deno.writeTextFile(exportJson, JSON.stringify(obj));
+    }
+
+    if (exportCsv) {
+      const f = await Deno.open(exportCsv, {
+        write: true,
+        create: true,
+        truncate: true,
+      });
+      const encoder = new TextEncoder();
+      const newline = new Uint8Array(1);
+      newline[0] = 0x0a;
+      await f.write(encoder.encode(columns.join(",")));
+      await f.write(newline);
+
+      for (let i = 0; i < values.length; i++) {
+        await f.write(encoder.encode(values[i].join(",")));
+        await f.write(newline);
+      }
+
+      f.close();
+    }
+  }
+  console.log("done");
+  return {
+    throughput: sum / time,
+  };
+}
+
+if (import.meta.main) {
+  await new Command()
+    .name("wmillbench")
+    .description("Run Benchmark to measure throughput of windmill.")
+    .version(VERSION)
+    .option("--host <url:string>", "The windmill host to benchmark.", {
+      default: "http://127.0.0.1:8000",
     })
-  )
-  .parse();
+    .option(
+      "--workers <workers:number>",
+      "The number of workers to run at once.",
+      {
+        default: 1,
+      }
+    )
+    .option(
+      "-s --seconds <seconds:number>",
+      "How long to run the benchmark for (in seconds).",
+      {
+        default: 30,
+      }
+    )
+    .option("--max <max:number>", "Maximum number of operations performed.")
+    .option("-e --email <email:string>", "The email to use to login.")
+    .option("-p --password <password:string>", "The password to use to login.")
+    .env(
+      "WM_TOKEN=<token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .option(
+      "-t --token <token:string>",
+      "The token to use when talking to the API server. Preferred over manual login."
+    )
+    .env(
+      "WM_WORKSPACE=<workspace:string>",
+      "The workspace to spawn scripts from."
+    )
+    .option(
+      "-w --workspace <workspace:string>",
+      "The workspace to spawn scripts from.",
+      { default: "admins" }
+    )
+    .option(
+      "-m --metrics <metrics:string>",
+      "The url to scrape metrics from.",
+      {
+        default: "http://localhost:8001/metrics",
+      }
+    )
+    .option(
+      "--export-json <export_json:string>",
+      "If set, exports will be into a JSON file."
+    )
+    .option(
+      "--export-csv <export_csv:string>",
+      "If set, exports will be into a csv file."
+    )
+    .option(
+      "--export-histograms <export_histograms:string[]>",
+      "Mark metrics (without label) that are reported as histograms to export."
+    )
+    .option(
+      "--export-simple <export_simple:string[]>",
+      "Mark metrics (without label) that are reported as simple values."
+    )
+    .option(
+      "--maximum-throughput <maximum_throughput:number>",
+      "Maximum number of jobs/flows to start in one second.",
+      {
+        default: Infinity,
+      }
+    )
+    .option("--use-flows", "Run flows instead of jobs.")
+    .option(
+      "--flow-pattern <pattern:string>",
+      "Use a different flow pattern among: 2steps, onebranch (Default 2steps)"
+    )
+    .option(
+      "--script-pattern <pattern:string>",
+      "Use a different script pattern among: deno, identity, python, go, bash (Default deno)"
+    )
+    .option("--custom <custom_path:string>", "Use custom actions during bench")
+    .option(
+      "--zombie-timeout <zombie_timeout:number>",
+      "The maximum time in ms to wait for jobs to complete.",
+      {
+        default: 90000,
+      }
+    )
+    .option(
+      "--continous",
+      "Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked."
+    )
+    .option(
+      "--histogram-buckets <histogram_buckets:string[]>",
+      "Define what buckets to collect from histograms.",
+      {
+        default: [
+          "+Inf",
+          "10",
+          "5",
+          "2.5",
+          "2.5",
+          "1",
+          "0.5",
+          "0.25",
+          "0.1",
+          "0.05",
+          "0.025",
+          "0.01",
+          "0.005",
+        ],
+      }
+    )
+    .option("--hide-progress", "Hide worker progress logs")
+    .action(main)
+    .command(
+      "upgrade",
+      new UpgradeCommand({
+        main: "main.ts",
+        args: [
+          "--allow-net",
+          "--allow-read",
+          "--allow-write",
+          "--allow-env",
+          "--unstable",
+        ],
+        provider: new DenoLandProvider({ name: "wmillbench" }),
+      })
+    )
+    .parse();
+}

benchmarks/plot.py

@@ -0,0 +1,21 @@
+import matplotlib.pyplot as plt
+import json
+
+with open("/tmp/windmill/profiling.json") as json_file:
+    data = json.load(json_file)
+    i = 0
+    for impacts in data["timings"][:]:
+        i += 1
+        if i % 200 != 0:
+            continue
+        timefilteredForce = plt.plot(impacts)
+        plt.text(len(impacts) - 1, impacts[-1], f"{i}")
+        timefilteredForce = plt.xlabel("points")
+        timefilteredForce = plt.ylabel("Force fast")
+
+    # for impacts in data["timings"][0:100]:
+    #     timefilteredForce = plt.plot(impacts)
+    #     timefilteredForce = plt.xlabel("points")
+    #     timefilteredForce = plt.ylabel("Force slow")
+
+    plt.show()

benchmarks/pulumi/.gitignore

@@ -1 +1,3 @@
-node_modules
+/bin/
+/node_modules/
+

benchmarks/pulumi/Pulumi.bench.yaml

@@ -0,0 +1,5 @@
+config:
+  aws:region: us-east-2
+  tailscale:apiKey:
+    secure: AAABAMGx6JDiUSM4/2M7ZLmws26KJYAGVmCb+BdURSGdO32scaf96Xy3Ocu/Ncv7G1y1W6K1EwZieN6L0/cTAZDGCJpRcMrMmuDyqE8u8Uuv2uRLKRA0
+  tailscale:tailnet: windmill.dev

benchmarks/pulumi/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: aws-bench
+runtime: nodejs
+description: Windmill Benchmarks on AWS

benchmarks/pulumi/index.ts

@@ -0,0 +1,391 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as aws from "@pulumi/aws";
+import * as awsx from "@pulumi/awsx";
+import * as tailscale from "@pulumi/tailscale";
+
+const vpc = new aws.ec2.Vpc("bench", { cidrBlock: "10.0.0.0/16" });
+
+// Create a security group allowing inbound access over port 80 and outbound
+// access to anywhere.
+const securityGroup = new aws.ec2.SecurityGroup("bench", {
+  vpcId: vpc.id,
+  ingress: [
+    {
+      cidrBlocks: ["0.0.0.0/0"],
+      protocol: "tcp",
+      fromPort: 80,
+      toPort: 50000,
+    },
+    {
+      cidrBlocks: ["0.0.0.0/0"],
+      protocol: "tcp",
+      fromPort: 22,
+      toPort: 22,
+    },
+  ],
+  egress: [
+    {
+      cidrBlocks: ["0.0.0.0/0"],
+      fromPort: 0,
+      toPort: 0,
+      protocol: "-1",
+    },
+  ],
+});
+
+// Create an an internet gateway.
+const gateway = new aws.ec2.InternetGateway("gw", {
+  vpcId: vpc.id,
+});
+
+// Find the latest Amazon Linux 2 AMI.
+const ami = pulumi.output(
+  aws.ec2.getAmi({
+    owners: ["amazon"],
+    mostRecent: true,
+    filters: [
+      { name: "description", values: ["Amazon Linux 2 *"] },
+      { name: "architecture", values: ["x86_64"] },
+    ],
+  })
+);
+
+// Create a route table.
+const routes = new aws.ec2.RouteTable("route", {
+  vpcId: vpc.id,
+  routes: [
+    {
+      cidrBlock: "0.0.0.0/0",
+      gatewayId: gateway.id,
+    },
+  ],
+});
+
+const subnetA = new aws.ec2.Subnet("bench-a", {
+  availabilityZone: "us-east-2a",
+  vpcId: vpc.id,
+  cidrBlock: "10.0.5.0/24",
+  mapPublicIpOnLaunch: true,
+  tags: {
+    Name: "bench",
+  },
+});
+
+const subnetB = new aws.ec2.Subnet("bench-b", {
+  availabilityZone: "us-east-2b",
+  vpcId: vpc.id,
+  cidrBlock: "10.0.6.0/24",
+  tags: {
+    Name: "bench",
+  },
+});
+
+const subnetC = new aws.ec2.Subnet("bench-c", {
+  availabilityZone: "us-east-2c",
+  vpcId: vpc.id,
+  cidrBlock: "10.0.7.0/24",
+  tags: {
+    Name: "bench",
+  },
+});
+
+new aws.ec2.RouteTableAssociation("bench-a", {
+  subnetId: subnetA.id,
+  routeTableId: routes.id,
+});
+
+new aws.ec2.RouteTableAssociation("bench-b", {
+  subnetId: subnetB.id,
+  routeTableId: routes.id,
+});
+
+new aws.ec2.RouteTableAssociation("bench-c", {
+  subnetId: subnetC.id,
+  routeTableId: routes.id,
+});
+
+const rdSubnet = new aws.rds.SubnetGroup("bench", {
+  subnetIds: [subnetA.id, subnetB.id],
+  tags: {
+    Name: "bench",
+  },
+});
+
+const db = new aws.rds.Instance("bench", {
+  vpcSecurityGroupIds: [securityGroup.id],
+  dbSubnetGroupName: rdSubnet.name,
+  allocatedStorage: 20,
+  dbName: "windmill",
+  engine: "postgres",
+  engineVersion: "14.8",
+  instanceClass: "db.m5d.large",
+  password: "postgres",
+  skipFinalSnapshot: true,
+  username: "postgres",
+  availabilityZone: "us-east-2a",
+  performanceInsightsEnabled: true,
+});
+
+const deployer = new aws.ec2.KeyPair("bench", {
+  publicKey:
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDRi2xytvAIgfs4f5B8EVrWH3bsu8CI+98iBazq08TuPEkLFP/ZqIVm8ZoTnpPadj2o0DXWHNjiyDAnGwp6W1rcSou4KCxoBViRX0doXBMat5GWs6Za2c+bmKllWUH2jhy4aPrdoI3il2URi7dIMH6WpWZDtD98CFK22rTgiBxhDydFpn3X1ULOI18f/SZEKzcb+iffhP5K/Sjqo65xPiKSl2m40vtaQjn2jTE7vmTD9c/dLv/5A3Sgh7CW2PJe9HR7/8qkzZkRkXtyODOIFfWN+vk0rGvk97IpV+DlSUag+FFeSK5Jn41w9IQV7frPvJAQOZ6eLq2GNv8iO/7QnfhlIUlPeClNJBRgb76FmiuslH+AOBiy/Szy3KUtoV/LO+uzvJxRZVX9DhSvTCm4D2qUTjozS+OSeBVuug70euWe29I7djouAuIbDOFkIkxStjhwISXfgvyWYVh8IMxC6+umZMhMff5CfqzpHY0jy2Tw5r/v/2Iz8EQXNRNAxnqKMPE= rfiszel@rubx1",
+});
+
+const sampleKey = new tailscale.TailnetKey("bench-ipn", {
+  ephemeral: false,
+  expiry: 60 * 60 * 24 * 90,
+  preauthorized: true,
+  reusable: true,
+});
+
+const ipn = new aws.ec2.Instance("bench-ipn-2", {
+  ami: ami.id,
+  availabilityZone: "us-east-2a",
+  keyName: deployer.keyName,
+  instanceType: "t3.medium",
+  subnetId: subnetA.id,
+  associatePublicIpAddress: true,
+  rootBlockDevice: {
+    volumeSize: 40,
+  },
+  vpcSecurityGroupIds: pulumi
+    .all([db.vpcSecurityGroupIds, securityGroup.id])
+    .apply(([db, sg]) => db.concat(sg)),
+  userData: pulumi.interpolate`#!/bin/bash
+sudo yum update -y
+sudo yum install -y yum-utils
+sudo yum-config-manager --add-repo https://pkgs.tailscale.com/stable/amazon-linux/2/tailscale.repo
+sudo yum install -y tailscale
+sudo systemctl enable --now tailscaled
+sleep 30
+sudo tailscale up --authkey="${sampleKey.key}" --hostname=bench-ipn --ssh --advertise-tags=tag:ipns
+
+sudo yum install -y wget unzip
+`,
+});
+
+// Find the latest Amazon Linux 2 AMI.
+const amiEcs = pulumi.output(
+  aws.ec2.getAmi({
+    owners: ["amazon"],
+    mostRecent: true,
+    filters: [{ name: "name", values: ["amzn2-ami-ecs-*-arm64-*"] }],
+  })
+);
+
+const clusterName = "windmill";
+const cluster = new aws.ecs.Cluster("cluster", { name: clusterName });
+
+const lt = new aws.ec2.LaunchTemplate("lt2", {
+  namePrefix: "windmill",
+  imageId: amiEcs.id,
+  keyName: deployer.keyName,
+  instanceType: "t4g.medium",
+  iamInstanceProfile: {
+    name: "ecsInstanceRole",
+  },
+
+  updateDefaultVersion: true,
+  vpcSecurityGroupIds: [securityGroup.id],
+  userData: btoa(`#!/bin/bash
+echo ECS_CLUSTER=${clusterName} >> /etc/ecs/ecs.config
+`),
+});
+
+const asg = new aws.autoscaling.Group("asg3", {
+  launchTemplate: {
+    id: lt.id,
+    version: "$Latest",
+  },
+  minSize: 0,
+  maxSize: 15,
+  vpcZoneIdentifiers: [subnetA.id],
+  tags: [
+    {
+      key: "AmazonECSManaged",
+      value: "true",
+      propagateAtLaunch: true,
+    },
+  ],
+});
+
+const capProvider = new aws.ecs.CapacityProvider("cap-provider", {
+  autoScalingGroupProvider: {
+    autoScalingGroupArn: asg.arn,
+    managedTerminationProtection: "DISABLED",
+    managedScaling: {
+      maximumScalingStepSize: 1000,
+      minimumScalingStepSize: 1,
+      status: "ENABLED",
+      targetCapacity: 80,
+    },
+  },
+});
+
+const clusterCapProvider = new aws.ecs.ClusterCapacityProviders(
+  "cap-provider",
+  {
+    clusterName: cluster.name,
+    capacityProviders: [capProvider.name],
+  }
+);
+
+const lb = new awsx.lb.ApplicationLoadBalancer("lb2", {
+  defaultTargetGroup: {
+    targetType: "instance",
+  },
+  subnetIds: [subnetA.id, subnetB.id, subnetC.id],
+});
+// const service = new awsx.ecs.FargateService("service", {
+//   cluster: cluster.arn,
+//   assignPublicIp: true,
+//   desiredCount: 2,
+//   taskDefinitionArgs: {
+//     container: {
+//       image: "nginx:latest",
+//       cpu: 512,
+//       memory: 128,
+//       essential: true,
+//       portMappings: [
+//         {
+//           targetGroup: lb.defaultTargetGroup,
+//         },
+//       ],
+//     },
+//   },
+// });
+
+db.address.apply((address) => {
+  const worker_td = new aws.ecs.TaskDefinition("worker-2", {
+    family: "windmill-worker",
+    containerDefinitions: JSON.stringify([
+      {
+        name: "windmill-worker",
+        image: "ghcr.io/windmill-labs/windmill:1.165.0",
+        cpu: 1024,
+        memory: 1800,
+        essential: true,
+        mountPaths: [
+          {
+            containerPath: "/tmp/windmill/cache",
+            sourceVolume: "dependency_cache",
+          },
+        ],
+        environment: [
+          { name: "DISABLE_SERVER", value: "true" },
+          // { name: "METRICS_ADDR", value: "true" },
+          { name: "RUST_LOG", value: "info" },
+          {
+            name: "DATABASE_URL",
+            value: `postgres://postgres:postgres@${address}/windmill?sslmode=disable`,
+          },
+        ],
+
+        logConfiguration: {
+          logDriver: "awslogs",
+          options: {
+            "awslogs-group": "windmill-worker",
+            "awslogs-region": "us-east-2",
+            "awslogs-create-group": "true",
+            "awslogs-stream-prefix": "windmill-worker",
+          },
+        },
+        dockerLabels: {
+          PROMETHEUS_EXPORTER_PORT: "8001",
+        },
+      },
+    ]),
+    volumes: [
+      {
+        name: "dependency_cache",
+        dockerVolumeConfiguration: {
+          scope: "shared",
+          autoprovision: true,
+        },
+      },
+    ],
+  });
+
+  const server_td = new aws.ecs.TaskDefinition("server", {
+    family: "windmill-server",
+    containerDefinitions: JSON.stringify([
+      {
+        name: "windmill-server",
+        image: "ghcr.io/windmill-labs/windmill:1.165.0",
+        cpu: 1024,
+        memory: 1024,
+        essential: true,
+        environment: [
+          { name: "NUM_WORKERS", value: "0" },
+          // { name: "METRICS_ADDR", value: "true" },
+          { name: "RUST_LOG", value: "info" },
+          {
+            name: "DATABASE_URL",
+            value: `postgres://postgres:postgres@${address}/windmill?sslmode=disable`,
+          },
+        ],
+        logConfiguration: {
+          logDriver: "awslogs",
+          options: {
+            "awslogs-group": "windmill-server",
+            "awslogs-region": "us-east-2",
+            "awslogs-create-group": "true",
+            "awslogs-stream-prefix": "windmill-server",
+          },
+        },
+        dockerLabels: {
+          PROMETHEUS_EXPORTER_PORT: "8001",
+        },
+        portMappings: [
+          {
+            containerPort: 8000,
+          },
+          {
+            containerPort: 8001,
+          },
+        ],
+      },
+    ]),
+  });
+
+  const service_server = new aws.ecs.Service("service-server", {
+    cluster: cluster.id,
+    taskDefinition: server_td.arn,
+    desiredCount: 2,
+    forceNewDeployment: true,
+    orderedPlacementStrategies: [
+      {
+        type: "binpack",
+        field: "cpu",
+      },
+    ],
+    loadBalancers: [
+      {
+        targetGroupArn: lb.defaultTargetGroup.arn,
+        containerName: "windmill-server",
+        containerPort: 8000,
+      },
+    ],
+  });
+
+  const service_worker = new aws.ecs.Service("service-worker", {
+    cluster: cluster.id,
+    taskDefinition: worker_td.arn,
+    desiredCount: 30,
+    forceNewDeployment: true,
+    orderedPlacementStrategies: [
+      {
+        type: "binpack",
+        field: "cpu",
+      },
+    ],
+  });
+});
+
+module.exports = {
+  databaseAddress: db.address,
+  ipn: ipn.publicIp,
+  url: lb.loadBalancer.dnsName,
+  amiEcs: amiEcs.id,
+  // instanceURL: pulumi.interpolate`http://${instance.publicIp}`,
+};

benchmarks/pulumi/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "aws-bench",
+  "main": "index.ts",
+  "devDependencies": {
+    "@types/node": "^16",
+    "pulumi": "^0.0.1"
+  },
+  "dependencies": {
+    "@pulumi/aws": "^5.0.0",
+    "@pulumi/awsx": "^1.0.4",
+    "@pulumi/pulumi": "^3.0.0",
+    "@pulumi/tailscale": "^0.12.2",
+    "@pulumi/tls": "^4.10.0"
+  }
+}