all

refresh superadmin state on workspace list page

.github/workflows/docker-image.yml

@@ -1,10 +1,8 @@
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME:
-    ${{ github.event_name != 'pull_request' && github.repository ||
+  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository ||
     'windmill-labs/windmill-test' }}
-  DEV_SHA:
-    ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
+  DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
     github.event.number) }}
 
 name: Build windmill:main
@@ -745,7 +743,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
-          file: "./docker/DockerfileRust"
+          file: "./docker/DockerfileFull"
           tags: |
             ${{ steps.meta-public.outputs.tags }}
           labels: |
@@ -790,7 +788,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
-          file: "./docker/DockerfileRustEe"
+          file: "./docker/DockerfileFullEe"
           tags: |
             ${{ steps.meta-ee-public.outputs.tags }}
           labels: |

CHANGELOG.md

@@ -1,5 +1,161 @@
 # Changelog
 
+## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01)
+
+
+### Bug Fixes
+
+* fix new instance db setup ([73ab8e1](https://github.com/windmill-labs/windmill/commit/73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86))
+
+## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01)
+
+
+### Features
+
+* flow step skipping ([#4461](https://github.com/windmill-labs/windmill/issues/4461)) ([0df169e](https://github.com/windmill-labs/windmill/commit/0df169e3f996ed54b91569b13cce15d7d019a213))
+
+
+### Bug Fixes
+
+* skip one migration to avoid using md5 for azure support ([630ae5d](https://github.com/windmill-labs/windmill/commit/630ae5d425cd9957d674befd2df96e2befec52a3))
+
+## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30)
+
+
+### Bug Fixes
+
+* improve allowed domains setting for sso ([24f4a7c](https://github.com/windmill-labs/windmill/commit/24f4a7caaafa93f51669dcf44a3dca09d5b228bb))
+
+## [1.402.2](https://github.com/windmill-labs/windmill/compare/v1.402.1...v1.402.2) (2024-09-28)
+
+
+### Bug Fixes
+
+* make form properties disablable ([0779d47](https://github.com/windmill-labs/windmill/commit/0779d47c1d39626d11bd3769cd787cb036df0a94))
+
+## [1.402.1](https://github.com/windmill-labs/windmill/compare/v1.402.0...v1.402.1) (2024-09-28)
+
+
+### Bug Fixes
+
+* allow preprocessor to write to args.json on nsjail ([#4455](https://github.com/windmill-labs/windmill/issues/4455)) ([0b9ec83](https://github.com/windmill-labs/windmill/commit/0b9ec83036e2a1d0773b4ec5856f907b383e9323))
+* **frontend:** Fix flow graph bg in dark mode on chrome ([#4454](https://github.com/windmill-labs/windmill/issues/4454)) ([6956a3a](https://github.com/windmill-labs/windmill/commit/6956a3a2ba6d189528cb34ab05f7137fdf4f840b))
+* improve suspend_first behavior and frequency ([b5e226b](https://github.com/windmill-labs/windmill/commit/b5e226b977e6d24ebd28bc1e7c867cb4888f77b2))
+
+## [1.402.0](https://github.com/windmill-labs/windmill/compare/v1.401.0...v1.402.0) (2024-09-26)
+
+
+### Features
+
+* **cli:** add queues, workers and worker-groups commands ([#4439](https://github.com/windmill-labs/windmill/issues/4439)) ([9f91b19](https://github.com/windmill-labs/windmill/commit/9f91b1995a98c9e096c6e599c4d5a8d5ea499ada))
+
+## [1.401.0](https://github.com/windmill-labs/windmill/compare/v1.400.0...v1.401.0) (2024-09-25)
+
+
+### Features
+
+* add return_last_result annotation to sql ([#4443](https://github.com/windmill-labs/windmill/issues/4443)) ([3ce5587](https://github.com/windmill-labs/windmill/commit/3ce5587faae3912ceedae4644732fa9704eb6d76))
+
+
+### Bug Fixes
+
+* fix flow rendering ([fd58e7e](https://github.com/windmill-labs/windmill/commit/fd58e7eb48c4fb66d199c33d0f8aaf2535485a2f))
+
+## [1.400.0](https://github.com/windmill-labs/windmill/compare/v1.399.0...v1.400.0) (2024-09-25)
+
+
+### Features
+
+* add static variable and resources support to ansible ([#4435](https://github.com/windmill-labs/windmill/issues/4435)) ([398a09b](https://github.com/windmill-labs/windmill/commit/398a09b7419c7adafa76e5f9f5a981bec6f0611d))
+* **frontend:** Catch flow errors in the UI ([#4429](https://github.com/windmill-labs/windmill/issues/4429)) ([84eefad](https://github.com/windmill-labs/windmill/commit/84eefadfcf06b4f38117b6c4c534f47fb2ef7cc7))
+
+
+### Bug Fixes
+
+* remove autocomplete for searchbar ([#4440](https://github.com/windmill-labs/windmill/issues/4440)) ([a1ac583](https://github.com/windmill-labs/windmill/commit/a1ac583f05c34a534c4ae78ca7d2a6850723ce85))
+
+## [1.399.0](https://github.com/windmill-labs/windmill/compare/v1.398.1...v1.399.0) (2024-09-25)
+
+
+### Features
+
+* add tag filtering to external JWT authentication ([#4425](https://github.com/windmill-labs/windmill/issues/4425)) ([590321f](https://github.com/windmill-labs/windmill/commit/590321fd3c88046a657e0a751b62a19424102060))
+
+
+### Bug Fixes
+
+* **cli:** version the whole client for deno compatibility ([81c2bb0](https://github.com/windmill-labs/windmill/commit/81c2bb069176a95b8fb9c52b31e5e03c1cc78afc))
+* correct AI generation for CRON and regex ([#4437](https://github.com/windmill-labs/windmill/issues/4437)) ([aeb5b5b](https://github.com/windmill-labs/windmill/commit/aeb5b5bcd163e2e7d1413d7e4aed8bd769996f24))
+
+## [1.398.1](https://github.com/windmill-labs/windmill/compare/v1.398.0...v1.398.1) (2024-09-23)
+
+
+### Bug Fixes
+
+* time handling in bun and deno ([#4423](https://github.com/windmill-labs/windmill/issues/4423)) ([61f27ac](https://github.com/windmill-labs/windmill/commit/61f27acbd5b477f0e2453d4b02d406a7aef76009))
+
+## [1.398.0](https://github.com/windmill-labs/windmill/compare/v1.397.4...v1.398.0) (2024-09-23)
+
+
+### Features
+
+* **frontend:** add http routing templates ([#4421](https://github.com/windmill-labs/windmill/issues/4421)) ([e99e7b2](https://github.com/windmill-labs/windmill/commit/e99e7b2b0bb1bc63d1253d147b80bac06eaff103))
+* http routing ([#4339](https://github.com/windmill-labs/windmill/issues/4339)) ([304dac3](https://github.com/windmill-labs/windmill/commit/304dac34475b8871621c54a768275fae1e9f845e))
+
+
+### Bug Fixes
+
+* allow no body in job requests ([#4413](https://github.com/windmill-labs/windmill/issues/4413)) ([70fa78c](https://github.com/windmill-labs/windmill/commit/70fa78c10d8afb3e06be84e3449fdfdeb2673765))
+* **frontend:** Fix delete branch one index ([#4418](https://github.com/windmill-labs/windmill/issues/4418)) ([30017cc](https://github.com/windmill-labs/windmill/commit/30017cc2b1401ea439e3fbec5d7764b206100a38))
+* migrate smtp instance settings to global settings ([#4416](https://github.com/windmill-labs/windmill/issues/4416)) ([50a6f78](https://github.com/windmill-labs/windmill/commit/50a6f789fa8a03d70407b0429da049cc0740b6fa))
+* no failed renewal alert if trial ([#4414](https://github.com/windmill-labs/windmill/issues/4414)) ([5be7be0](https://github.com/windmill-labs/windmill/commit/5be7be03a6df34a9fae96ed7f844d693f7cfce3e))
+* update pip to 24.2 and python 3.11.8->3.11.10 ([a17195d](https://github.com/windmill-labs/windmill/commit/a17195d88ff52b72f0c2cdd95c6218df8ef6d7cf))
+* various improvements for ansible ([#4419](https://github.com/windmill-labs/windmill/issues/4419)) ([a500994](https://github.com/windmill-labs/windmill/commit/a500994cc2cd9cc6cbad722059cebd0bf74a8850))
+
+## [1.397.4](https://github.com/windmill-labs/windmill/compare/v1.397.3...v1.397.4) (2024-09-20)
+
+
+### Bug Fixes
+
+* **cli:** make CLI not require latest version of windmill ([794f87a](https://github.com/windmill-labs/windmill/commit/794f87aa0d33a0b782fcb2726754fdc51bc1d73d))
+
+## [1.397.3](https://github.com/windmill-labs/windmill/compare/v1.397.2...v1.397.3) (2024-09-20)
+
+
+### Bug Fixes
+
+* **cli:** make CLI compatible with Node 18 ([8212532](https://github.com/windmill-labs/windmill/commit/8212532b29fbcf971cb320b8cfc84f3a8d8ad795))
+
+## [1.397.2](https://github.com/windmill-labs/windmill/compare/v1.397.1...v1.397.2) (2024-09-20)
+
+
+### Bug Fixes
+
+* **bun:** never pre-bundle BASE_URL env variable ([69b8754](https://github.com/windmill-labs/windmill/commit/69b8754aef32682bf812be783f79b81cd8526c7a))
+
+## [1.397.1](https://github.com/windmill-labs/windmill/compare/v1.397.0...v1.397.1) (2024-09-20)
+
+
+### Bug Fixes
+
+* **go:** put shared directory in job dir and not go parent ([623ece8](https://github.com/windmill-labs/windmill/commit/623ece8c6791b2c18648a68c562fcd5c8291be9b))
+* improve ability to paste from macos in vscode extension ([07372e7](https://github.com/windmill-labs/windmill/commit/07372e7e65d77df338718929f133e477a1daf993))
+* update git sync script ([30fe28c](https://github.com/windmill-labs/windmill/commit/30fe28ceecb9043eee6811dce8c3e2cd9224927b))
+
+## [1.397.0](https://github.com/windmill-labs/windmill/compare/v1.396.1...v1.397.0) (2024-09-18)
+
+
+### Features
+
+* ansible playbook support ([#4399](https://github.com/windmill-labs/windmill/issues/4399)) ([6855b8d](https://github.com/windmill-labs/windmill/commit/6855b8da9ad92aed514ee1ab214c9550d04e7a22))
+
+## [1.396.1](https://github.com/windmill-labs/windmill/compare/v1.396.0...v1.396.1) (2024-09-18)
+
+
+### Bug Fixes
+
+* postgres scripts that take longer than 20s do not timeout anymore ([37d152f](https://github.com/windmill-labs/windmill/commit/37d152feeb6abeb061daa5c93c24916e520b1fd0))
+
 ## [1.396.0](https://github.com/windmill-labs/windmill/compare/v1.395.0...v1.396.0) (2024-09-17)
 
 

Dockerfile

@@ -1,6 +1,6 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
 ARG RUST_IMAGE=rust:1.80-slim-bookworm
-ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
+ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
 
@@ -95,6 +95,8 @@ ARG WITH_KUBECTL=true
 ARG WITH_HELM=true
 ARG WITH_GIT=true
 
+RUN pip install --upgrade pip==24.2
+
 RUN apt-get update \
     && apt-get install -y ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1  software-properties-common \
     && apt-get clean \

backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json

@@ -0,0 +1,89 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441"
+}

backend/.sqlx/query-03d63d2e64b012f624d2731b5bcb8849c74a9474777be61edf0ed43ddda07ef3.json

@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "created_by",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "format_extension",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -46,6 +51,7 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json

@@ -57,7 +57,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json

@@ -71,7 +71,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n            WHERE id = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90"
-}

backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET \n        ping_at = now(), \n        jobs_executed = 1, \n        current_job_id = $1, \n        current_job_workspace_id = 'admins' \n        WHERE worker = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182"
+}

backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json

@@ -0,0 +1,36 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63"
+}

backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n                WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f"
+}

backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_at FROM metrics WHERE id = 'telemetry' ORDER BY created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604"
+}

backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json

@@ -51,7 +51,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe"
+}

backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838"
+}

backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return \n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
+  "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor \n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -15,8 +15,8 @@
       },
       {
         "ordinal": 2,
-        "name": "early_return",
-        "type_info": "Text"
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -31,5 +31,5 @@
       null
     ]
   },
-  "hash": "872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e"
+  "hash": "3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d"
 }

backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
+  "query": "INSERT INTO metrics (id, value)\n                            VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n                            FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
   "describe": {
     "columns": [],
     "parameters": {
@@ -11,5 +11,5 @@
     },
     "nullable": []
   },
-  "hash": "9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5"
+  "hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a"
 }

backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f"
+}

backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE http_trigger \n                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n                WHERE workspace_id = $11 AND path = $12",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea"
+}

backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json

@@ -0,0 +1,108 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n            FROM http_trigger\n            WHERE workspace_id = $1 AND path = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path_key",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "edited_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 10,
+        "name": "extra_perms",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 11,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491"
+}

backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json

@@ -0,0 +1,36 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n                WHERE workspace_id = $9 AND path = $10",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0"
+}

backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2",
+  "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -56,7 +56,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }
@@ -81,6 +82,11 @@
         "ordinal": 10,
         "name": "timeout",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 11,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -100,8 +106,9 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c"
+  "hash": "64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac"
 }

backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_try_advisory_xact_lock($1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "pg_try_advisory_xact_lock",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f"
+}

backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -65,21 +65,36 @@
       },
       {
         "ordinal": 12,
+        "name": "occupancy_rate_15s",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 13,
+        "name": "occupancy_rate_5m",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 14,
+        "name": "occupancy_rate_30m",
+        "type_info": "Float4"
+      },
+      {
+        "ordinal": 15,
         "name": "memory",
         "type_info": "Int8"
       },
       {
-        "ordinal": 13,
+        "ordinal": 16,
         "name": "vcpus",
         "type_info": "Int8"
       },
       {
-        "ordinal": 14,
+        "ordinal": 17,
         "name": "memory_usage",
         "type_info": "Int8"
       },
       {
-        "ordinal": 15,
+        "ordinal": 18,
         "name": "wm_memory_usage",
         "type_info": "Int8"
       }
@@ -108,8 +123,11 @@
       true,
       true,
       true,
+      true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af"
+  "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab"
 }

backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                                SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n                                last_ping = NULL\n                                WHERE id = $2\n                                RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46"
+}

backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json

@@ -0,0 +1,73 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "job_kind",
+            "kind": {
+              "Enum": [
+                "script",
+                "preview",
+                "flow",
+                "dependencies",
+                "flowpreview",
+                "script_hub",
+                "identity",
+                "flowdependencies",
+                "http",
+                "graphql",
+                "postgresql",
+                "noop",
+                "appdependencies",
+                "deploymentcallback",
+                "singlescriptflow"
+              ]
+            }
+          }
+        },
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Timestamptz",
+        "Varchar",
+        "Jsonb",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561"
+}

backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c"
+  "hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0"
 }

backend/.sqlx/query-7b1239ad6460e8f5fb41bfe12f662a779528784ec8cf3f6dcce5545ab90bf234.json

@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "created_by",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "format_extension",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -45,6 +50,7 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json

@@ -0,0 +1,45 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar",
+        "Varchar",
+        "Int8",
+        {
+          "Custom": {
+            "name": "script_lang",
+            "kind": {
+              "Enum": [
+                "python3",
+                "deno",
+                "go",
+                "bash",
+                "postgresql",
+                "nativets",
+                "bun",
+                "mysql",
+                "bigquery",
+                "snowflake",
+                "graphql",
+                "powershell",
+                "mssql",
+                "php",
+                "bunnative",
+                "rust",
+                "ansible"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf"
+}

backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json

@@ -1,23 +1,22 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
+  "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
         "name": "?column?",
-        "type_info": "Jsonb"
+        "type_info": "Bool"
       }
     ],
     "parameters": {
       "Left": [
-        "Uuid",
-        "Int4"
+        "Uuid"
       ]
     },
     "nullable": [
       null
     ]
   },
-  "hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4"
+  "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3"
 }

backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json

@@ -1,11 +1,11 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT config FROM config WHERE name = 'server'",
+  "query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
-        "name": "config",
+        "name": "value",
         "type_info": "Jsonb"
       }
     ],
@@ -13,8 +13,8 @@
       "Left": []
     },
     "nullable": [
-      true
+      false
     ]
   },
-  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+  "hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b"
 }

backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json

@@ -0,0 +1,37 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd"
+}

backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n                WHERE id = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368"
+}

backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json

@@ -0,0 +1,41 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor\n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "early_return",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      null,
+      null
+    ]
+  },
+  "hash": "98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba"
+}

backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb"
+}

backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb"
+}

backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json

@@ -1,21 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "TextArray",
-        "Float4",
-        "Int8",
-        "Int8",
-        "Text",
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51"
-}

backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json

@@ -56,7 +56,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -51,7 +51,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef"
+}

backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n                 occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n                 memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "TextArray",
+        "Float4",
+        "Int8",
+        "Int8",
+        "Text",
+        "Int8",
+        "Int8",
+        "Float4",
+        "Float4",
+        "Float4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
+}

backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n                WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa"
+}

backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -37,7 +37,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }
@@ -71,10 +72,11 @@
         "Varchar",
         "Bool",
         "Bool",
-        "Varchar"
+        "Varchar",
+        "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb"
+  "hash": "b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3"
 }

backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f"
+}

backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n            WHERE id = $4",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9"
-}

backend/.sqlx/query-b8d392ccfcccafe0c19511b3567bc11779b1052b0948c410468a8aeba1d26d33.json

@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "created_by",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "format_extension",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -45,6 +50,7 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                    SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n                    WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772"
+}

backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json

@@ -0,0 +1,37 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8"
+}

backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n        FROM completed_job\n        WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n        GROUP BY tag\n        ORDER BY \"count!\" DESC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Float8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb"
+}

backend/.sqlx/query-cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "format_extension",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "cf1cef7e0fe2e7e3db96b0ec005360361b9eec023a6fc2a4a7a917f59d86af4d"
+}

backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                        SET last_ping = null\n                        WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12"
-}

backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json

@@ -77,7 +77,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }

backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "int4",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43"
-}

backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188"
+}

backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
+  "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n                            occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
   "describe": {
     "columns": [],
     "parameters": {
@@ -9,10 +9,14 @@
         "Varchar",
         "Int8",
         "Int8",
-        "Text"
+        "Text",
+        "Float4",
+        "Float4",
+        "Float4",
+        "Float4"
       ]
     },
     "nullable": []
   },
-  "hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4"
+  "hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
 }

backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json

@@ -32,6 +32,11 @@
         "ordinal": 5,
         "name": "created_by",
         "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "format_extension",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -43,6 +48,7 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                            SET flow_status = JSONB_SET(\n                                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n                                ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n                                ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n                            ),\n                            last_ping = NULL\n                            WHERE id = $2\n                            RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "int4",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Uuid",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b"
+}

backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len  FROM queue WHERE id = $1",
+  "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
   "describe": {
     "columns": [
       {
@@ -24,5 +24,5 @@
       null
     ]
   },
-  "hash": "f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8"
+  "hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929"
 }

backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
+  "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
   "describe": {
     "columns": [
       {
@@ -56,7 +56,8 @@
                 "mssql",
                 "php",
                 "bunnative",
-                "rust"
+                "rust",
+                "ansible"
               ]
             }
           }
@@ -81,6 +82,11 @@
         "ordinal": 10,
         "name": "timeout",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 11,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -100,8 +106,9 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99"
+  "hash": "fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a"
 }

backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json

@@ -0,0 +1,87 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183"
+}

backend/.sqlx/query-ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO resource_type\n            (workspace_id, name, schema, description, created_by, edited_at)\n            VALUES ($1, $2, $3, $4, $5, now())",
+  "query": "INSERT INTO resource_type\n            (workspace_id, name, schema, description, created_by, format_extension, edited_at)\n            VALUES ($1, $2, $3, $4, $5, $6, now())",
   "describe": {
     "columns": [],
     "parameters": {
@@ -9,10 +9,11 @@
         "Varchar",
         "Jsonb",
         "Text",
+        "Varchar",
         "Varchar"
       ]
     },
     "nullable": []
   },
-  "hash": "d249343a1d9fce2f7ca9e6e447eb32d7d11b61acdab80dda24db4a9ba3e1f7e6"
+  "hash": "ffedbb3a2676a6d7b71f81f89109a02a8dba90d40144e942527f8a3fc36dfbc1"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.396.0"
+version = "1.403.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -23,10 +23,11 @@ members = [
     "./parsers/windmill-parser-py",
     "./parsers/windmill-parser-py-imports",
     "./parsers/windmill-sql-datatype-parser-wasm",
+    "./parsers/windmill-parser-yaml",
 ]
 
 [workspace.package]
-version = "1.396.0"
+version = "1.403.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -43,7 +44,7 @@ default = []
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml"]
 stripe = ["windmill-api/stripe"]
-benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
+benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
 pg_embed = ["dep:pg-embed"]
@@ -118,6 +119,7 @@ windmill-parser-py = { path = "./parsers/windmill-parser-py" }
 windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" }
 windmill-parser-go = { path = "./parsers/windmill-parser-go" }
 windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
+windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" }
 windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
 windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
 windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
@@ -250,6 +252,7 @@ indexmap = { version = "2.2.5", features = ["serde"]}
 tokio-native-tls = "^0"
 openssl = "=0.10"
 mail-parser = "^0"
+matchit = "=0.7.3"
 
 datafusion = "39.0.0"
 object_store = { version = "0.10.0", features = ["aws", "azure"] }
@@ -277,3 +280,4 @@ toml = "0.7"
 syn = { version = "2.0.74", features = ["full"] }
 quote = "1.0.36"
 regex-lite = "0.1.6"
+yaml-rust = "0.4.5"

backend/custom_migrations/create_workspace_without_md5.sql

@@ -0,0 +1,16 @@
+INSERT INTO workspace(id, name, owner) VALUES
+	('admins', 'Admins', 'admin@windmill.dev') ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_settings (workspace_id) VALUES
+	('admins') ON CONFLICT DO NOTHING;
+
+INSERT INTO workspace_key
+	(workspace_id, kind, key)
+	VALUES ('admins', 'cloud', array_to_string(
+    array(
+        SELECT chr( (trunc(65 + random() * 25)::int) + 
+                    CASE WHEN random() > 0.5 THEN 32 ELSE 0 END )  -- generates random uppercase/lowercase letters
+        FROM generate_series(1, 32) -- generates 32 characters
+    ),
+    ''
+)) ON CONFLICT DO NOTHING;

backend/ee-repo-ref.txt

@@ -1 +1 @@
-534f50aae2f89281304b3c3ba943d18afc07fad4
+0f5f42d2f8f5f1af05c8086f3dc7ad38d83750df

backend/migrations/20240906082526_add_ansible_lang.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240906082526_add_ansible_lang.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'ansible';
+UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["ansible"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust"]}'::jsonb AND NOT config->'worker_tags' @> '"ansible"'::jsonb;

backend/migrations/20240910151523_resource_type_file_format.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+ALTER TABLE resource_type
+  DROP COLUMN format_extension;

backend/migrations/20240910151523_resource_type_file_format.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE resource_type
+  ADD COLUMN format_extension VARCHAR(20);

backend/migrations/20240917172503_job_routing.down.sql

@@ -0,0 +1,7 @@
+-- Add down migration script here
+DROP TABLE http_trigger;
+DROP TYPE http_method;
+
+ALTER TABLE script DROP COLUMN has_preprocessor;
+
+DROP FUNCTION prevent_route_path_change();

backend/migrations/20240917172503_job_routing.up.sql

@@ -0,0 +1,61 @@
+-- Add up migration script here
+CREATE TYPE http_method AS ENUM ('get', 'post', 'put', 'delete', 'patch');
+
+CREATE TABLE http_trigger (
+  path VARCHAR(255) NOT NULL,
+  route_path VARCHAR(255) NOT NULL,
+  route_path_key VARCHAR(255) NOT NULL,
+  script_path VARCHAR(255) NOT NULL,
+  is_flow BOOLEAN NOT NULL,
+  workspace_id VARCHAR(50) NOT NULL,
+  edited_by VARCHAR(50) NOT NULL,
+  email VARCHAR(255) NOT NULL,
+  edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  extra_perms JSONB NOT NULL DEFAULT '{}',
+  is_async BOOLEAN NOT NULL DEFAULT FALSE,
+  requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
+  http_method http_method NOT NULL,
+  PRIMARY KEY (path, workspace_id)
+);
+
+
+GRANT SELECT, UPDATE ON http_trigger TO windmill_user;
+GRANT ALL ON http_trigger TO windmill_admin;
+
+ALTER TABLE http_trigger ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY see_folder_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
+USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
+CREATE POLICY see_folder_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
+USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+
+CREATE POLICY see_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
+CREATE POLICY see_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
+USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups_select ON http_trigger FOR SELECT TO windmill_user
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
+CREATE POLICY see_extra_perms_groups_update ON http_trigger FOR UPDATE TO windmill_user
+USING (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));
+
+CREATE OR REPLACE FUNCTION prevent_route_path_change()
+RETURNS TRIGGER AS $$
+BEGIN
+    IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN
+        RAISE EXCEPTION 'Modification of route_path is only allowed by admins';
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER check_route_path_change
+BEFORE UPDATE ON http_trigger
+FOR EACH ROW
+EXECUTE FUNCTION prevent_route_path_change();
+
+ALTER TABLE script ADD COLUMN has_preprocessor BOOLEAN;

backend/migrations/20240919200009_migrate_smtp_settings.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240919200009_migrate_smtp_settings.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+INSERT INTO global_settings (name, value)
+SELECT 'smtp_settings', config
+FROM config
+WHERE name = 'server'
+ON CONFLICT (name) DO UPDATE SET value = excluded.value;
+
+DELETE FROM config WHERE name = 'server';

backend/migrations/20240924213010_add_last_occupancy_rates.down.sql

@@ -0,0 +1,5 @@
+-- Add down migration script here
+ALTER TABLE worker_ping
+DROP COLUMN occupancy_rate_15s,
+DROP COLUMN occupancy_rate_5m,
+DROP COLUMN occupancy_rate_30m;

backend/migrations/20240924213010_add_last_occupancy_rates.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE worker_ping
+ADD COLUMN occupancy_rate_15s REAL,
+ADD COLUMN occupancy_rate_5m REAL,
+ADD COLUMN occupancy_rate_30m REAL;

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -16,7 +16,13 @@ pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_bash_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing bash script".to_string()))
     }
@@ -26,7 +32,13 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_powershell_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing powershell script".to_string()))
     }
@@ -173,7 +185,8 @@ non_required="${5:-}"
                         oidx: None
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-go/src/lib.rs

@@ -42,6 +42,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -49,6 +50,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -241,7 +243,8 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
                         oidx: None
                     },
                 ],
-                no_main_func: Some(false)
+                no_main_func: Some(false),
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-graphql/src/lib.rs

@@ -15,7 +15,13 @@ pub fn parse_graphql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_graphql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -119,7 +125,8 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") {
                         oidx: None
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-php/src/lib.rs

@@ -82,6 +82,7 @@ pub fn parse_php_signature(
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -89,6 +90,7 @@ pub fn parse_php_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -159,7 +161,8 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f
                         oidx: None
                     }
                 ],
-                no_main_func: Some(false)
+                no_main_func: Some(false),
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-py/src/lib.rs

@@ -63,9 +63,17 @@ pub fn parse_python_signature(
 ) -> anyhow::Result<MainArgSignature> {
     let main_name = override_main.unwrap_or("main".to_string());
 
+    let has_preprocessor = !filter_non_main(code, "preprocessor").is_empty();
+
     let filtered_code = filter_non_main(code, &main_name);
     if filtered_code.is_empty() {
-        return Err(anyhow::anyhow!("No main function found".to_string(),));
+        return Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: vec![],
+            no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
+        });
     }
     let ast = Suite::parse(&filtered_code, "main.py")
         .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
@@ -119,6 +127,7 @@ pub fn parse_python_signature(
                 })
                 .collect(),
             no_main_func: Some(false),
+            has_preprocessor: Some(has_preprocessor),
         })
     } else {
         Ok(MainArgSignature {
@@ -126,6 +135,7 @@ pub fn parse_python_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
         })
     }
 }
@@ -325,6 +335,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
                     },
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -389,6 +400,7 @@ def main(test1: str,
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -448,6 +460,7 @@ def main(test1: str,
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -491,6 +504,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -521,6 +535,59 @@ def main(test1: DynSelect_foo): return
                     oidx: None
                 }],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_6() -> anyhow::Result<()> {
+        let code = r#"
+
+import os
+
+def hello(): return
+
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code, None)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![],
+                no_main_func: Some(true),
+                has_preprocessor: Some(false)
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_7() -> anyhow::Result<()> {
+        let code = r#"
+
+import os
+
+def preprocessor(): return
+
+def main(): return
+
+
+
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code, None)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![],
+                no_main_func: Some(false),
+                has_preprocessor: Some(true)
             }
         );
 

backend/parsers/windmill-parser-rust/src/lib.rs

@@ -29,6 +29,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -36,6 +37,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -455,10 +457,7 @@ fn main(
         assert_eq!(ret.args[2].typ, Typ::Str(None));
 
         assert_eq!(ret.args[3].name, "my_string_vec");
-        assert_eq!(
-            ret.args[3].otyp,
-            Some("Vec < String >".to_string())
-        );
+        assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string()));
         assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None))));
     }
 

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -20,7 +20,13 @@ pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_mysql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -30,7 +36,13 @@ pub fn parse_pgsql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_pg_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -40,7 +52,13 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_bigquery_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -50,7 +68,13 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_snowflake_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -60,7 +84,13 @@ pub fn parse_mssql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_mssql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -570,7 +600,8 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
                         oidx: Some(2),
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -618,7 +649,8 @@ SELECT $2::TEXT;
                         oidx: Some(3),
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -744,7 +776,8 @@ SELECT ?, ?;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -791,7 +824,8 @@ SELECT :param2;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -830,7 +864,8 @@ SELECT @token;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -877,7 +912,8 @@ SELECT ?;
                         oidx: None,
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -924,7 +960,8 @@ SELECT @p2;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -159,6 +159,14 @@ pub fn parse_deno_signature(
         })?
         .body;
 
+    let has_preprocessor = ast.iter().any(|x| match x {
+        ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
+            decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, .. }),
+            ..
+        })) => &sym.to_string() == "preprocessor",
+        _ => false,
+    });
+
     let main_name = main_override.unwrap_or("main".to_string());
     let params = ast.into_iter().find_map(|x| match x {
         ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
@@ -178,6 +186,7 @@ pub fn parse_deno_signature(
                 .map(|x| parse_param(x, &cm, skip_dflt, &mut c))
                 .collect::<anyhow::Result<Vec<Arg>>>()?,
             no_main_func: Some(false),
+            has_preprocessor: Some(has_preprocessor),
         };
         Ok(r)
     } else {
@@ -186,6 +195,7 @@ pub fn parse_deno_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
         })
     }
 }

backend/parsers/windmill-parser-wasm/Cargo.toml

@@ -25,6 +25,7 @@ ts-parser = [ "dep:windmill-parser-ts"]
 php-parser = [ "dep:windmill-parser-php"]
 rust-parser = [ "dep:windmill-parser-rust"]
 graphql-parser = [ "dep:windmill-parser-graphql"]
+ansible-parser = [ "dep:windmill-parser-yaml"]
 
 [dependencies]
 anyhow.workspace = true
@@ -37,6 +38,7 @@ windmill-parser-ts = { workspace = true, optional = true }
 windmill-parser-php = { workspace = true, optional = true }
 windmill-parser-graphql = { workspace = true, optional = true }
 windmill-parser-rust = { workspace = true, optional = true }
+windmill-parser-yaml = { workspace = true, optional = true }
 wasm-bindgen.workspace = true
 serde_json.workspace = true
 getrandom = { workspace = true, features = ["js"] }

backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh

@@ -10,35 +10,41 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \
 OUT_DIR="pkg-ts"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
 
 # sql languages, graphql and bash/powershell, since they all use regex
 OUT_DIR="pkg-regex"
 wasm-pack build --release --target web --out-dir $OUT_DIR \
 	--features "sql-parser,graphql-parser,bash-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
 
 # python
 OUT_DIR="pkg-py"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
 
 # go
 OUT_DIR="pkg-go"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
 
 # php
 OUT_DIR="pkg-php"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
 
 # rust
 OUT_DIR="pkg-rust"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
+
+# ansible
+OUT_DIR="pkg-yaml"
+wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \
+	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json

backend/parsers/windmill-parser-wasm/build-pkgs.sh

@@ -42,3 +42,9 @@ OUT_DIR="pkg-rust"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
 sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
+
+# ansible
+OUT_DIR="pkg-yaml"
+wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \
+	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
+sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json

backend/parsers/windmill-parser-wasm/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eou pipefail
 
-deno task wasmbuild --out ../../../cli/wasm/  -p windmill-parser-wasm
+deno task wasmbuild --out ../../../cli/wasm/  -p windmill-parser-wasm --all-features

backend/parsers/windmill-parser-wasm/publish-pkgs.sh

@@ -21,3 +21,6 @@ popd
 
 pushd "pkg-rust" && npm publish ${args}
 popd
+
+pushd "pkg-yaml" && npm publish ${args}
+popd

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -17,8 +17,12 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
 
 #[cfg(feature = "ts-parser")]
 #[wasm_bindgen]
-pub fn parse_deno(code: &str) -> String {
-    wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None))
+pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
+    wrap_sig(windmill_parser_ts::parse_deno_signature(
+        code,
+        false,
+        main_override,
+    ))
 }
 
 #[cfg(feature = "ts-parser")]
@@ -65,8 +69,11 @@ pub fn parse_go(code: &str) -> String {
 
 #[cfg(feature = "py-parser")]
 #[wasm_bindgen]
-pub fn parse_python(code: &str) -> String {
-    wrap_sig(windmill_parser_py::parse_python_signature(code, None))
+pub fn parse_python(code: &str, main_override: Option<String>) -> String {
+    wrap_sig(windmill_parser_py::parse_python_signature(
+        code,
+        main_override,
+    ))
 }
 
 #[cfg(feature = "sql-parser")]
@@ -122,3 +129,9 @@ pub fn parse_php(code: &str) -> String {
 pub fn parse_rust(code: &str) -> String {
     wrap_sig(windmill_parser_rust::parse_rust_signature(code))
 }
+
+#[cfg(feature = "ansible-parser")]
+#[wasm_bindgen]
+pub fn parse_ansible(code: &str) -> String {
+    wrap_sig(windmill_parser_yaml::parse_ansible_sig(code))
+}

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -139,7 +139,8 @@ export function main(test1?: string, test2: string = \"burkina\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -215,7 +216,8 @@ export function main(test2 = \"burkina\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -264,7 +266,8 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -294,7 +297,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
                 has_default: false,
                 oidx: None
             }],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -434,6 +438,7 @@ Write-Output 'Testing...'
                 }
             ],
             no_main_func: None,
+            has_preprocessor: None
         }
     );
 

backend/parsers/windmill-parser-yaml/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "windmill-parser-yaml"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[lib]
+name = "windmill_parser_yaml"
+path = "./src/lib.rs"
+
+[dependencies]
+yaml-rust.workspace = true
+windmill-parser.workspace = true
+anyhow.workspace = true
+serde_json.workspace = true

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -0,0 +1,482 @@
+use anyhow::anyhow;
+use serde_json::json;
+use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
+use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
+
+pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature> {
+    let docs = YamlLoader::load_from_str(inner_content)
+        .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?;
+
+    if docs.len() < 2 {
+        return Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: vec![],
+            no_main_func: None,
+            has_preprocessor: None,
+        });
+    }
+
+    let mut args = vec![];
+    if let Yaml::Hash(doc) = &docs[0] {
+        for (key, value) in doc {
+            match key {
+                Yaml::String(key) if key == "extra_vars" => {
+                    if let Yaml::Hash(v) = &value {
+                        for (key, arg) in v {
+                            if let Yaml::String(arg_name) = key {
+                                // if not then it is a static resource or var, so skip
+                                // it will be fetched by the worker
+                                if let ArgTyp::Typ(typ) = parse_ansible_arg_typ(arg) {
+                                    let default = get_default_for_typ(&arg);
+                                    args.push(Arg {
+                                        name: arg_name.to_string(),
+                                        otyp: None,
+                                        typ,
+                                        has_default: default.is_some(),
+                                        default,
+                                        oidx: None,
+                                    })
+                                }
+                            }
+                        }
+                    }
+                }
+                Yaml::String(key) if key == "inventory" => {
+                    for inv in parse_inventories(value)? {
+                        if inv.pinned_resource.is_some() {
+                            continue;
+                        }
+                        args.push(Arg {
+                            name: inv.name,
+                            otyp: None,
+                            typ: Typ::Resource(inv.resource_type.unwrap_or("Unknown".to_string())),
+                            has_default: inv.default.is_some(),
+                            default: inv.default.map(|v| json!(format!("$res:{}", v))),
+                            oidx: None,
+                        })
+                    }
+                }
+                _ => (),
+            }
+        }
+    }
+    Ok(MainArgSignature {
+        star_args: false,
+        star_kwargs: false,
+        args,
+        no_main_func: None,
+        has_preprocessor: None,
+    })
+}
+
+fn get_default_for_typ(arg: &Yaml) -> Option<serde_json::Value> {
+    if let Yaml::Hash(arg) = arg {
+        if let Some(def) = arg.get(&Yaml::String("default".to_string())) {
+            if let Some(Yaml::String(typ)) = arg.get(&Yaml::String("type".to_string())) {
+                if let Yaml::String(path) = def {
+                    if typ.as_str() == "windmill_resource" {
+                        return Some(serde_json::Value::String(format!("$res:{}", path)));
+                    }
+                }
+            }
+            return Some(yaml_to_json(def));
+        }
+    }
+    None
+}
+
+enum ArgTyp {
+    Typ(Typ),
+    StaticVar(String),
+    StaticResource(String),
+}
+
+fn parse_ansible_arg_typ(arg: &Yaml) -> ArgTyp {
+    if let Yaml::Hash(a) = arg {
+        if let Some(Yaml::String(typ)) = a.get(&Yaml::String("type".to_string())) {
+            if typ.as_str() == "windmill_variable" {
+                if let Some(Yaml::String(variable_path)) =
+                    a.get(&Yaml::String("variable".to_string()))
+                {
+                    return ArgTyp::StaticVar(variable_path.to_string());
+                }
+            }
+            if typ.as_str() == "windmill_resource" {
+                if let Some(Yaml::String(resource_path)) =
+                    a.get(&Yaml::String("resource".to_string()))
+                {
+                    return ArgTyp::StaticResource(resource_path.to_string());
+                }
+            }
+        }
+    }
+    ArgTyp::Typ(parse_ansible_typ(arg))
+}
+
+fn parse_ansible_typ(arg: &Yaml) -> Typ {
+    if let Yaml::Hash(arg) = arg {
+        if let Some(Yaml::String(typ)) = arg.get(&Yaml::String("type".to_string())) {
+            match typ.as_ref() {
+                "boolean" => Typ::Bool,
+                "integer" => Typ::Int,
+                "number" => Typ::Float,
+                "string" => {
+                    if let Some(Yaml::String(fmt)) = arg.get(&Yaml::String("format".to_string())) {
+                        match fmt.as_ref() {
+                            "date-time" | "datetime" => Typ::Datetime,
+                            "email" => Typ::Email,
+                            _ => Typ::Str(None),
+                        }
+                    } else {
+                        Typ::Str(None)
+                    }
+                }
+                "object" => {
+                    if let Some(Yaml::Hash(props)) =
+                        arg.get(&Yaml::String("properties".to_string()))
+                    {
+                        let mut prop_vec = vec![];
+                        for (key, value) in props {
+                            if let Yaml::String(key) = key {
+                                prop_vec.push(ObjectProperty {
+                                    key: key.clone(),
+                                    typ: Box::new(parse_ansible_typ(value)),
+                                })
+                            }
+                        }
+                        Typ::Object(prop_vec)
+                    } else {
+                        Typ::Object(vec![])
+                    }
+                }
+                "array" => {
+                    if let Some(items) = arg.get(&Yaml::String("items".to_string())) {
+                        Typ::List(Box::new(parse_ansible_typ(items)))
+                    } else {
+                        Typ::List(Box::new(Typ::Unknown))
+                    }
+                }
+                "windmill_resource" => {
+                    if let Some(Yaml::String(res_name)) =
+                        arg.get(&Yaml::String("resource_type".to_string()))
+                    {
+                        Typ::Resource(res_name.clone())
+                    } else {
+                        Typ::Resource("".to_string())
+                    }
+                }
+                _ => Typ::Unknown,
+            }
+        } else {
+            Typ::Unknown
+        }
+    } else {
+        Typ::Unknown
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AnsiblePlaybookOptions {
+    // There are a lot more options
+    // TODO: Add the options as customers require them
+    // Add it here and then on the executors cmd_options
+    pub verbosity: Option<String>,
+    pub forks: Option<i64>,
+    pub timeout: Option<i64>,
+    pub flush_cache: Option<()>,
+    pub force_handlers: Option<()>,
+}
+
+#[derive(Debug, Clone)]
+pub struct FileResource {
+    pub resource_path: String,
+    pub target_path: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct AnsibleInventory {
+    pub default: Option<String>,
+    pub name: String,
+    resource_type: Option<String>,
+    pub pinned_resource: Option<String>,
+}
+#[derive(Debug, Clone)]
+pub struct AnsibleRequirements {
+    pub python_reqs: Vec<String>,
+    pub collections: Option<String>,
+    pub file_resources: Vec<FileResource>,
+    pub inventories: Vec<AnsibleInventory>,
+    pub vars: Vec<(String, String)>,
+    pub resources: Vec<(String, String)>,
+    pub options: AnsiblePlaybookOptions,
+}
+
+fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInventory>> {
+    if let Yaml::Array(arr) = inventory_yaml {
+        let mut ret = vec![];
+        for (i, inv) in arr.iter().enumerate() {
+            if let Yaml::Hash(inv) = inv {
+                let resource_type = inv
+                    .get(&Yaml::String("resource_type".to_string()))
+                    .and_then(|v| v.as_str())
+                    .map(|s| s.to_string());
+
+                let windmill_path = inv
+                    .get(&Yaml::String("default".to_string()))
+                    .and_then(|v| v.as_str())
+                    .map(|s| s.to_string());
+
+                let name = if i == 0 {
+                    "inventory.ini".to_string()
+                } else {
+                    format!("inventory-{}.ini", i)
+                };
+
+                let name = inv
+                    .get(&Yaml::String("name".to_string()))
+                    .and_then(|v| v.as_str())
+                    .unwrap_or(name.as_str())
+                    .to_string();
+
+                let pin = inv
+                    .get(&Yaml::String("resource".to_string()))
+                    .and_then(|v| v.as_str())
+                    .map(|s| s.to_string());
+
+                ret.push(AnsibleInventory {
+                    default: windmill_path,
+                    name,
+                    resource_type,
+                    pinned_resource: pin,
+                });
+            }
+        }
+        return Ok(ret);
+    }
+    return Err(anyhow!("Invalid inventory definition"));
+}
+
+pub fn parse_ansible_reqs(
+    inner_content: &str,
+) -> anyhow::Result<(String, Option<AnsibleRequirements>, String)> {
+    let mut logs = String::new();
+    let docs = YamlLoader::load_from_str(inner_content)
+        .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?;
+
+    if docs.len() < 2 {
+        return Ok((logs, None, inner_content.to_string()));
+    }
+
+    let opts = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
+    let mut ret = AnsibleRequirements {
+        python_reqs: vec![],
+        collections: None,
+        file_resources: vec![],
+        inventories: vec![],
+        vars: vec![],
+        resources: vec![],
+        options: opts,
+    };
+
+    if let Yaml::Hash(doc) = &docs[0] {
+        for (key, value) in doc {
+            match key {
+                Yaml::String(key) if key == "dependencies" => {
+                    if let Yaml::Hash(deps) = value {
+                        if let Some(galaxy_requirements) =
+                            deps.get(&Yaml::String("galaxy".to_string()))
+                        {
+                            let mut out_str = String::new();
+                            let mut emitter = YamlEmitter::new(&mut out_str);
+                            emitter.dump(galaxy_requirements)?;
+                            ret.collections = Some(out_str);
+                        }
+                        if let Some(Yaml::Array(py_reqs)) =
+                            deps.get(&Yaml::String("python".to_string()))
+                        {
+                            ret.python_reqs = py_reqs
+                                .iter()
+                                .map(|d| d.as_str().map(|s| s.to_string()))
+                                .filter_map(|x| x)
+                                .collect();
+                        }
+                    }
+                }
+                Yaml::String(key) if key == "file_resources" => {
+                    if let Yaml::Array(file_resources) = value {
+                        let resources: anyhow::Result<Vec<FileResource>> =
+                            file_resources.iter().map(parse_file_resource).collect();
+                        ret.file_resources.append(&mut resources?);
+                    }
+                }
+                Yaml::String(key) if key == "extra_vars" => {
+                    if let Yaml::Hash(v) = &value {
+                        for (key, arg) in v {
+                            if let Yaml::String(arg_name) = key {
+                                match parse_ansible_arg_typ(arg) {
+                                    ArgTyp::StaticVar(p) => {
+                                        ret.vars
+                                            .push((arg_name.to_string(), format!("$var:{}", p)));
+                                    }
+                                    ArgTyp::StaticResource(p) => {
+                                        ret.resources
+                                            .push((arg_name.to_string(), format!("$res:{}", p)));
+                                    }
+                                    ArgTyp::Typ(_) => (),
+                                }
+                            }
+                        }
+                    }
+                }
+                Yaml::String(key) if key == "inventory" => {
+                    ret.inventories = parse_inventories(value)?;
+                }
+                Yaml::String(key) if key == "options" => {
+                    if let Yaml::Array(opts) = &value {
+                        ret.options = parse_ansible_options(opts);
+                    }
+                }
+                Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
+                _ => (),
+            }
+        }
+    }
+    let mut out_str = String::new();
+    let mut emitter = YamlEmitter::new(&mut out_str);
+
+    for i in 1..docs.len() {
+        emitter.dump(&docs[i])?;
+    }
+    Ok((logs, Some(ret), out_str))
+}
+
+fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
+    let mut ret = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
+    for opt in opts {
+        if let Yaml::String(o) = opt {
+            match o.as_str() {
+                "flush_cache" => {
+                    ret.flush_cache = Some(());
+                }
+                "force_handlers" => {
+                    ret.force_handlers = Some(());
+                }
+                s if count_consecutive_vs(s) > 0 => {
+                    ret.verbosity = Some("v".repeat(count_consecutive_vs(s).min(6)));
+                }
+                _ => (),
+            }
+        }
+        if let Yaml::Hash(m) = opt {
+            if let Some((Yaml::String(name), value)) = m.iter().last() {
+                match name.as_str() {
+                    "forks" => {
+                        if let Yaml::Integer(count) = value {
+                            ret.forks = Some(*count);
+                        }
+                    }
+                    "timeout" => {
+                        if let Yaml::Integer(timeout) = value {
+                            ret.timeout = Some(*timeout);
+                        }
+                    }
+                    "verbosity" => {
+                        if let Yaml::String(verbosity) = value {
+                            let c = count_consecutive_vs(verbosity);
+                            if c > 0 && c <= 6 {
+                                ret.verbosity = Some("v".repeat(c.min(6)));
+                            }
+
+                        }
+                    }
+                    _ => ()
+
+                }
+            }
+
+
+        }
+    }
+
+    ret
+}
+
+fn count_consecutive_vs(s: &str) -> usize {
+    let mut max_count = 0;
+    let mut current_count = 0;
+
+    for c in s.chars() {
+        if c == 'v' {
+            current_count += 1;
+            if current_count == 6 {
+                return 6;  // Stop early if we reach 6
+            }
+        } else {
+            current_count = 0;  // Reset count if the character is not 'v'
+        }
+        max_count = max_count.max(current_count);
+    }
+
+    max_count
+}
+
+fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
+    if let Yaml::Hash(f) = yaml {
+        if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) {
+            let target_path = f
+                .get(&Yaml::String("target".to_string()))
+                .and_then(|x| x.as_str())
+                .map(|x| x.to_string())
+                .ok_or(anyhow!(
+                    "No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.",
+                    resource_path
+                ))?;
+            return Ok(FileResource { resource_path: resource_path.clone(), target_path });
+        }
+        return Err(anyhow!(
+            "File resource should have a `resource` field, linking to a text file resource"
+        ));
+    }
+    return Err(anyhow!("Invalid file resource: Should be a dictionary."));
+}
+
+fn yaml_to_json(yaml: &Yaml) -> serde_json::Value {
+    match yaml {
+        Yaml::Array(arr) => {
+            let json_array: Vec<serde_json::Value> = arr.into_iter().map(yaml_to_json).collect();
+            serde_json::Value::Array(json_array)
+        }
+        Yaml::Hash(hash) => {
+            let json_object = hash
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = match k {
+                        Yaml::String(s) => s.clone(),
+                        _ => k.as_str().unwrap_or("").to_string(),
+                    };
+                    (key, yaml_to_json(v))
+                })
+                .collect();
+            serde_json::Value::Object(json_object)
+        }
+        Yaml::String(s) => serde_json::Value::String(s.to_string()),
+        Yaml::Integer(i) => serde_json::Value::Number(i.clone().into()),
+        Yaml::Real(r) => serde_json::Value::Number(r.parse().unwrap_or(0.into())),
+        Yaml::Boolean(b) => serde_json::Value::Bool(*b),
+        Yaml::Null => serde_json::Value::Null,
+        _ => serde_json::Value::Null,
+    }
+}

backend/parsers/windmill-parser/src/lib.rs

@@ -16,6 +16,7 @@ pub struct MainArgSignature {
     pub star_kwargs: bool,
     pub args: Vec<Arg>,
     pub no_main_func: Option<bool>,
+    pub has_preprocessor: Option<bool>,
 }
 
 #[derive(Serialize, Clone, Debug, PartialEq)]
@@ -153,7 +154,10 @@ mod test {
 
     #[test]
     fn test_mixed_delimiters() {
-        assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces");
+        assert_eq!(
+            to_snake_case("test-Case_with Spaces"),
+            "test_case_with_spaces"
+        );
     }
 
     #[test]

backend/plot.py

@@ -0,0 +1,89 @@
+import json
+import matplotlib.pyplot as plt
+
+# Function to load JSON data from a file
+def load_json_data(filepath):
+    with open(filepath, 'r') as file:
+        data = json.load(file)
+    return data
+
+# Function to plot two arrays of subarrays with tuples (step_name, duration)
+def plot_two_arrays_of_subarrays(arrays1, arrays2):
+    # Function to calculate sum of durations for each step
+    def calculate_sums(arrays):
+        steps = [step for step, _ in arrays[0]['timings']]  # Extract steps from the first iteration
+        sums = {step: 0.0 for step in steps}  # Initialize sums dictionary with step names
+
+        # Sum up the durations for each step across all subarrays
+        for subarray in arrays:
+            for step_name, duration in subarray['timings']:
+                if step_name not in sums:
+                    sums[step_name] = 0
+                sums[step_name] += duration
+
+        for step_name, duration in sums.items():
+            sums[step_name] = duration / 1000000000
+
+        # Convert the sums dictionary to two lists (for plotting)
+        step_names = list(sums.keys())
+        durations = list(sums.values())
+        return step_names, durations
+
+    # Calculate sums for both arrays of subarrays
+    step_names1, sums1 = calculate_sums(arrays1)
+    step_names2, sums2 = calculate_sums(arrays2)
+
+    # Create two subplots, one on top of the other
+    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))
+
+    # First plot (top) for the first array of subarrays
+    ax1.bar(step_names1, sums1, color='b')
+    ax1.set_title('Total Duration per Step - Main Loop')
+    ax1.set_xlabel('Step Name')
+    ax1.set_ylabel('Total Duration (s)')
+    ax1.grid(True, axis='y')
+    ax1.tick_params(axis='x', rotation=45)
+
+    # Second plot (bottom) for the second array of subarrays
+    ax2.bar(step_names2, sums2, color='r')
+    ax2.set_title('Total Duration per Step - Result Processor')
+    ax2.set_xlabel('Step Name')
+    ax2.set_ylabel('Total Duration (s)')
+    ax2.grid(True, axis='y')
+    ax2.tick_params(axis='x', rotation=45)
+
+    # Adjust layout so the plots don't overlap
+    plt.tight_layout()
+
+    # Display the plot
+    plt.show()
+
+# Load arrays from the JSON files
+main = load_json_data('/tmp/windmill/profiling_main.json')
+result_processor = load_json_data('/tmp/windmill/profiling_result_processor.json')
+
+arrays1 = main['timings']
+arrays2 = result_processor['timings']
+
+total_duration1 = main['total_duration']/1000
+total_duration2 = result_processor['total_duration']/1000
+
+print(f"Total duration for main: {total_duration1}s")
+print(f"Total duration for result processor: {total_duration2}s")
+
+iterations_total = sum(main['iter_durations']) / 1000000000
+iterations_total2 = sum(result_processor['iter_durations']) / 1000000000
+
+print(f"Number of iterations: {len(main['iter_durations'])}")
+print(f"Total iterations for main: {iterations_total}s")
+print(f"Total iterations for result processor: {iterations_total2}s")
+
+# Calculate RPS
+rps1 = len(main['iter_durations']) / total_duration1
+rps2 = len(result_processor['iter_durations']) / total_duration2
+
+print(f"RPS for main: {rps1}")
+print(f"RPS for result processor: {rps2}")
+
+# Plot the data
+plot_two_arrays_of_subarrays(arrays1, arrays2)

backend/src/ee.rs

@@ -1,16 +0,0 @@
-use anyhow::anyhow;
-#[cfg(feature = "enterprise")]
-use windmill_common::error::{Error, Result};
-
-pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> {
-    // Implementation is not open source
-    Err(anyhow!("License cannot be set in Windmill CE"))
-}
-
-#[cfg(feature = "enterprise")]
-pub async fn verify_license_key() -> Result<()> {
-    // Implementation is not open source
-    Err(Error::InternalErr(
-        "License always invalid in Windmill CE".to_string(),
-    ))
-}

backend/src/ee.rs

@@ -0,0 +1 @@
+/git/windmill/../windmill-ee-private/src/ee.rs

backend/src/main.rs

@@ -8,7 +8,10 @@
 
 use anyhow::Context;
 use git_version::git_version;
-use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store};
+use monitor::{
+    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
+    send_logs_to_object_store,
+};
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
@@ -35,7 +38,7 @@ use windmill_common::{
         JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
         OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -74,7 +77,7 @@ use crate::monitor::{
     reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
     reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
     reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
-    reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
+    reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
     reload_worker_config,
 };
 
@@ -155,6 +158,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                 "global",
                 "global",
                 "",
+                &mut None,
             )
             .await?;
             tokio::fs::remove_dir_all(job_dir).await?;
@@ -173,8 +177,9 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
                     None,
                     &job_dir,
                     "cache_init",
-                    windmill_worker::get_common_bun_proc_envs("").await,
+                    windmill_worker::get_common_bun_proc_envs(None).await,
                     false,
+                    &mut None,
                 )
                 .await?;
             } else {
@@ -563,9 +568,7 @@ Windmill Community Edition {GIT_VERSION}
                                         "notify_config_change" => {
                                             match n.payload() {
                                                 "server" if server_mode => {
-                                                    tracing::info!("Server config change detected: {}", n.payload());
-
-                                                    reload_server_config(&db).await;
+                                                    tracing::error!("Server config change detected but server config is obsolete: {}", n.payload());
                                                 },
                                                 a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
                                                     tracing::info!("Worker config change detected: {}", n.payload());
@@ -609,6 +612,12 @@ Windmill Community Edition {GIT_VERSION}
                                                         tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
                                                     }
                                                 }
+                                                SMTP_SETTING => {
+                                                    reload_smtp_config(&db).await;
+                                                },
+                                                TIMEOUT_WAIT_RESULT_SETTING => {
+                                                    reload_timeout_wait_result_setting(&db).await
+                                                },
                                                 RETENTION_PERIOD_SECS_SETTING => {
                                                     reload_retention_period_setting(&db).await
                                                 },
@@ -736,9 +745,8 @@ Windmill Community Edition {GIT_VERSION}
             Ok(()) as anyhow::Result<()>
         };
 
-        let instance_name = rd_string(8);
         if mode == Mode::Server || mode == Mode::Standalone {
-            schedule_stats(instance_name, &db, &HTTP_CLIENT).await;
+            schedule_stats(&db, &HTTP_CLIENT).await;
         }
 
         #[cfg(feature = "enterprise")]

backend/src/monitor.rs

@@ -22,6 +22,7 @@ use tokio::{
 #[cfg(feature = "embedding")]
 use windmill_api::embeddings::update_embeddings_db;
 use windmill_api::{
+    jobs::TIMEOUT_WAIT_RESULT,
     oauth2_ee::{build_oauth_clients, OAuthClient},
     DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
 };
@@ -40,17 +41,17 @@ use windmill_common::{
         KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
         PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     jobs::QueuedJob,
     oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
-    server::load_server_config,
+    server::load_smtp_config,
     tracing_init::JSON_FMT,
     users::truncate_token,
     utils::{now_from_db, rd_string, report_critical_error, Mode},
     worker::{
         load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
-        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
+        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG,
         WORKER_GROUP,
     },
     BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
@@ -172,8 +173,9 @@ pub async fn initial_load(
         reload_s3_cache_setting(&db).await;
     }
 
+    reload_smtp_config(&db).await;
+
     if server_mode {
-        reload_server_config(&db).await;
         reload_retention_period_setting(&db).await;
         reload_request_size(&db).await;
         reload_saml_metadata_setting(&db).await;
@@ -523,7 +525,7 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
             ok_lines = counter.non_error_count;
             err_lines = counter.error_count;
         } else {
-            println!("no counter found for {ts_str}");
+            // println!("no counter found for {ts_str}");
         }
     } else {
         println!("Error reading log counters 2");
@@ -698,6 +700,15 @@ pub async fn reload_scim_token_setting(db: &DB) {
         .await;
 }
 
+pub async fn reload_timeout_wait_result_setting(db: &DB) {
+    reload_option_setting_with_tracing(
+        db,
+        TIMEOUT_WAIT_RESULT_SETTING,
+        "TIMEOUT_WAIT_RESULT",
+        TIMEOUT_WAIT_RESULT.clone(),
+    )
+    .await;
+}
 pub async fn reload_saml_metadata_setting(db: &DB) {
     reload_option_setting_with_tracing(
         db,
@@ -1060,83 +1071,78 @@ pub async fn monitor_db(
 }
 
 pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
-    let tx = db.begin().await;
-    if let Ok(mut tx) = tx {
-        let last_check = sqlx::query_scalar!(
+    let last_check = sqlx::query_scalar!(
             "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1"
         )
         .fetch_optional(db)
         .await
         .unwrap_or(Some(chrono::Utc::now()));
 
-        let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
-        let save_metrics = last_check
-            .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
-            .unwrap_or(true);
+    let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
+    let save_metrics = last_check
+        .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
+        .unwrap_or(true);
 
-        if metrics_enabled || save_metrics {
-            let queue_counts = sqlx::query!(
-                "SELECT tag, count(*) as count FROM queue WHERE
+    if metrics_enabled || save_metrics {
+        let queue_counts = sqlx::query!(
+            "SELECT tag, count(*) as count FROM queue WHERE
                 scheduled_for <= now() - ('3 seconds')::interval AND running = false
                 GROUP BY tag"
-            )
-            .fetch_all(&mut *tx)
-            .await
-            .ok()
-            .unwrap_or_else(|| vec![]);
+        )
+        .fetch_all(db)
+        .await
+        .ok()
+        .unwrap_or_else(|| vec![]);
 
-            for q in queue_counts {
-                let count = q.count.unwrap_or(0);
-                let tag = q.tag;
-                if metrics_enabled {
-                    let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
-                    metric.set(count as i64);
-                }
+        for q in queue_counts {
+            let count = q.count.unwrap_or(0);
+            let tag = q.tag;
+            if metrics_enabled {
+                let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
+                metric.set(count as i64);
+            }
 
-                // save queue_count and delay metrics per tag
-                if save_metrics {
+            // save queue_count and delay metrics per tag
+            if save_metrics {
+                sqlx::query!(
+                    "INSERT INTO metrics (id, value) VALUES ($1, $2)",
+                    format!("queue_count_{}", tag),
+                    serde_json::json!(count)
+                )
+                .execute(db)
+                .await
+                .ok();
+                if count > 0 {
                     sqlx::query!(
-                        "INSERT INTO metrics (id, value) VALUES ($1, $2)",
-                        format!("queue_count_{}", tag),
-                        serde_json::json!(count)
-                    )
-                    .execute(&mut *tx)
-                    .await
-                    .ok();
-                    if count > 0 {
-                        sqlx::query!(
                             "INSERT INTO metrics (id, value)
                             VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)
                             FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval
-                            ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
+                            ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
                             format!("queue_delay_{}", tag),
                             tag
-                        ).execute(&mut *tx).await.ok();
-                    }
+                        ).execute(db).await.ok();
                 }
             }
         }
-
-        // clean queue metrics older than 14 days
-        sqlx::query!(
-            "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
-        )
-        .execute(&mut *tx)
-        .await
-        .ok();
-
-        tx.commit().await.ok();
     }
+
+    // clean queue metrics older than 14 days
+    sqlx::query!(
+        "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
+    )
+    .execute(db)
+    .await
+    .ok();
 }
 
-pub async fn reload_server_config(db: &Pool<Postgres>) {
-    let config = load_server_config(&db).await;
-    if let Err(e) = config {
-        tracing::error!("Error reloading server config: {:?}", e)
+pub async fn reload_smtp_config(db: &Pool<Postgres>) {
+    let smtp_config = load_smtp_config(&db).await;
+    if let Err(e) = smtp_config {
+        tracing::error!("Error reloading smtp config: {:?}", e)
     } else {
-        let mut wc = SERVER_CONFIG.write().await;
-        tracing::info!("Reloading server config...");
-        *wc = config.unwrap()
+        let mut wc = SMTP_CONFIG.write().await;
+        tracing::info!("Reloading smtp config...");
+        *wc = smtp_config.unwrap()
     }
 }
 
@@ -1362,6 +1368,8 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             rsmq.clone(),
             worker_name,
             send_result_never_used,
+            #[cfg(feature = "benchmark")]
+            &mut windmill_common::bench::BenchmarkIter::new(),
         )
         .await;
     }

backend/tests/worker.rs

@@ -1126,6 +1126,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1166,6 +1167,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                             priority: None,
                             delete_after_use: None,
                             continue_on_error: None,
+                            skip_if: None,
                         }],
                     }
                     .into(),
@@ -1181,6 +1183,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
             ],
             same_worker: false,
@@ -1286,6 +1289,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "b".to_string(),
@@ -1336,6 +1340,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
+                                skip_if: None,
                             },
                             FlowModule {
                                 id: "e".to_string(),
@@ -1372,7 +1377,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                 priority: None,
                                 delete_after_use: None,
                                 continue_on_error: None,
-
+                                skip_if: None,
                             },
                         ],
                     }.into(),
@@ -1388,6 +1393,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
                 FlowModule {
                     id: "c".to_string(),
@@ -1431,6 +1437,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                     priority: None,
                     delete_after_use: None,
                     continue_on_error: None,
+                    skip_if: None,
                 },
             ],
             same_worker: true,
@@ -1912,6 +1919,109 @@ def main():
     assert_eq!(result, serde_json::json!("test-workspace"));
 }
 
+#[sqlx::test(fixtures("base"))]
+async fn test_bun_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Bun,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_deno_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Deno,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+from datetime import datetime
+def main(a: datetime, b: bytes):
+    return (isinstance(a, datetime), isinstance(b, bytes))
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Python3,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .arg("b", json!("dGVzdA=="))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!([true, true]));
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_empty_loop_1(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -3294,6 +3404,7 @@ async fn run_deployed_relative_imports(
                 visible_to_runner_only: None,
                 no_main_func: None,
                 codebase: None,
+                has_preprocessor: None,
             },
         )
         .await
@@ -3324,6 +3435,7 @@ async fn run_deployed_relative_imports(
                 dedicated_worker: None,
                 language,
                 priority: None,
+                apply_preprocessor: false,
             })
             .push(&db2)
             .await;

backend/windmill-api/Cargo.toml

@@ -94,9 +94,9 @@ object_store = { workspace = true, optional = true}
 openidconnect = { workspace = true, optional = true}
 url = { workspace = true, optional = true}
 jsonwebtoken = { workspace = true }
+matchit.workspace = true
 
 pin-project.workspace = true
-crc.workspace = true
 http.workspace = true
 async-stream.workspace = true
 ulid.workspace = true