noopener noreferrer

* Add static variable and resources to ansible

* Add wasm parser to cli

* Fix pulling of resources, rs and playbooks

* Remove unused import, add publish pkg-yaml

* Update package.json

* Update generate-metadata

* Fix null fields being serialized in script metadata

* npm install

.github/workflows/docker-image.yml

@@ -1,10 +1,8 @@
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME:
-    ${{ github.event_name != 'pull_request' && github.repository ||
+  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository ||
     'windmill-labs/windmill-test' }}
-  DEV_SHA:
-    ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
+  DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
     github.event.number) }}
 
 name: Build windmill:main

CHANGELOG.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## [1.399.0](https://github.com/windmill-labs/windmill/compare/v1.398.1...v1.399.0) (2024-09-25)
+
+
+### Features
+
+* add tag filtering to external JWT authentication ([#4425](https://github.com/windmill-labs/windmill/issues/4425)) ([590321f](https://github.com/windmill-labs/windmill/commit/590321fd3c88046a657e0a751b62a19424102060))
+
+
+### Bug Fixes
+
+* **cli:** version the whole client for deno compatibility ([81c2bb0](https://github.com/windmill-labs/windmill/commit/81c2bb069176a95b8fb9c52b31e5e03c1cc78afc))
+* correct AI generation for CRON and regex ([#4437](https://github.com/windmill-labs/windmill/issues/4437)) ([aeb5b5b](https://github.com/windmill-labs/windmill/commit/aeb5b5bcd163e2e7d1413d7e4aed8bd769996f24))
+
+## [1.398.1](https://github.com/windmill-labs/windmill/compare/v1.398.0...v1.398.1) (2024-09-23)
+
+
+### Bug Fixes
+
+* time handling in bun and deno ([#4423](https://github.com/windmill-labs/windmill/issues/4423)) ([61f27ac](https://github.com/windmill-labs/windmill/commit/61f27acbd5b477f0e2453d4b02d406a7aef76009))
+
+## [1.398.0](https://github.com/windmill-labs/windmill/compare/v1.397.4...v1.398.0) (2024-09-23)
+
+
+### Features
+
+* **frontend:** add http routing templates ([#4421](https://github.com/windmill-labs/windmill/issues/4421)) ([e99e7b2](https://github.com/windmill-labs/windmill/commit/e99e7b2b0bb1bc63d1253d147b80bac06eaff103))
+* http routing ([#4339](https://github.com/windmill-labs/windmill/issues/4339)) ([304dac3](https://github.com/windmill-labs/windmill/commit/304dac34475b8871621c54a768275fae1e9f845e))
+
+
+### Bug Fixes
+
+* allow no body in job requests ([#4413](https://github.com/windmill-labs/windmill/issues/4413)) ([70fa78c](https://github.com/windmill-labs/windmill/commit/70fa78c10d8afb3e06be84e3449fdfdeb2673765))
+* **frontend:** Fix delete branch one index ([#4418](https://github.com/windmill-labs/windmill/issues/4418)) ([30017cc](https://github.com/windmill-labs/windmill/commit/30017cc2b1401ea439e3fbec5d7764b206100a38))
+* migrate smtp instance settings to global settings ([#4416](https://github.com/windmill-labs/windmill/issues/4416)) ([50a6f78](https://github.com/windmill-labs/windmill/commit/50a6f789fa8a03d70407b0429da049cc0740b6fa))
+* no failed renewal alert if trial ([#4414](https://github.com/windmill-labs/windmill/issues/4414)) ([5be7be0](https://github.com/windmill-labs/windmill/commit/5be7be03a6df34a9fae96ed7f844d693f7cfce3e))
+* update pip to 24.2 and python 3.11.8->3.11.10 ([a17195d](https://github.com/windmill-labs/windmill/commit/a17195d88ff52b72f0c2cdd95c6218df8ef6d7cf))
+* various improvements for ansible ([#4419](https://github.com/windmill-labs/windmill/issues/4419)) ([a500994](https://github.com/windmill-labs/windmill/commit/a500994cc2cd9cc6cbad722059cebd0bf74a8850))
+
 ## [1.397.4](https://github.com/windmill-labs/windmill/compare/v1.397.3...v1.397.4) (2024-09-20)
 
 

Dockerfile

@@ -1,6 +1,6 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
 ARG RUST_IMAGE=rust:1.80-slim-bookworm
-ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
+ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
 
@@ -95,6 +95,8 @@ ARG WITH_KUBECTL=true
 ARG WITH_HELM=true
 ARG WITH_GIT=true
 
+RUN pip install --upgrade pip==24.2
+
 RUN apt-get update \
     && apt-get install -y ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1  software-properties-common \
     && apt-get clean \

backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json

@@ -0,0 +1,89 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441"
+}

backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n            WHERE id = $3",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90"
-}

backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET \n        ping_at = now(), \n        jobs_executed = 1, \n        current_job_id = $1, \n        current_job_workspace_id = 'admins' \n        WHERE worker = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182"
+}

backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json

@@ -0,0 +1,36 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63"
+}

backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n                WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f"
+}

backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return \n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
+  "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor \n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -15,8 +15,8 @@
       },
       {
         "ordinal": 2,
-        "name": "early_return",
-        "type_info": "Text"
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -31,5 +31,5 @@
       null
     ]
   },
-  "hash": "872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e"
+  "hash": "3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d"
 }

backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f"
+}

backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE http_trigger \n                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n                WHERE workspace_id = $11 AND path = $12",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea"
+}

backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json

@@ -0,0 +1,108 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n            FROM http_trigger\n            WHERE workspace_id = $1 AND path = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path_key",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "edited_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 10,
+        "name": "extra_perms",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 11,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491"
+}

backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json

@@ -0,0 +1,36 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n                WHERE workspace_id = $9 AND path = $10",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0"
+}

backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2",
+  "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -82,6 +82,11 @@
         "ordinal": 10,
         "name": "timeout",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 11,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -101,8 +106,9 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c"
+  "hash": "64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac"
 }

backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "bool",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json

@@ -1,11 +1,11 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT config FROM config WHERE name = 'server'",
+  "query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
-        "name": "config",
+        "name": "value",
         "type_info": "Jsonb"
       }
     ],
@@ -13,8 +13,8 @@
       "Left": []
     },
     "nullable": [
-      true
+      false
     ]
   },
-  "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
+  "hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b"
 }

backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json

@@ -0,0 +1,37 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd"
+}

backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n                WHERE id = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368"
+}

backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json

@@ -0,0 +1,41 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor\n        FROM flow \n        LEFT JOIN flow_version\n            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n        WHERE flow.path = $1 and flow.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tag",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "dedicated_worker",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "early_return",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true,
+      true,
+      null,
+      null
+    ]
+  },
+  "hash": "98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba"
+}

backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb"
+}

backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb"
+}

backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef"
+}

backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                SET flow_status = JSONB_SET(\n                    JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n                WHERE id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa"
+}

backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json

@@ -18,8 +18,8 @@
       "Left": []
     },
     "nullable": [
-      false,
-      true
+      true,
+      false
     ]
   },
   "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76"

backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)",
+  "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -72,10 +72,11 @@
         "Varchar",
         "Bool",
         "Bool",
-        "Varchar"
+        "Varchar",
+        "Bool"
       ]
     },
     "nullable": []
   },
-  "hash": "8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb"
+  "hash": "b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3"
 }

backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f"
+}

backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json

@@ -1,17 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue\n            SET flow_status = JSONB_SET(\n                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n            WHERE id = $4",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Jsonb",
-        "Jsonb",
-        "Uuid"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9"
-}

backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue\n                    SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n                    WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772"
+}

backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json

@@ -0,0 +1,37 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Bool",
+        {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8"
+}

backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188"
+}

backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len  FROM queue WHERE id = $1",
+  "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
   "describe": {
     "columns": [
       {
@@ -24,5 +24,5 @@
       null
     ]
   },
-  "hash": "f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8"
+  "hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929"
 }

backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
+  "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND\n    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n    deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
   "describe": {
     "columns": [
       {
@@ -82,6 +82,11 @@
         "ordinal": 10,
         "name": "timeout",
         "type_info": "Int4"
+      },
+      {
+        "ordinal": 11,
+        "name": "has_preprocessor",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -101,8 +106,9 @@
       true,
       true,
       true,
+      true,
       true
     ]
   },
-  "hash": "e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99"
+  "hash": "fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a"
 }

backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json

@@ -0,0 +1,87 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "script_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_flow",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "route_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "is_async",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "requires_auth",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 7,
+        "name": "edited_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "http_method: _",
+        "type_info": {
+          "Custom": {
+            "name": "http_method",
+            "kind": {
+              "Enum": [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch"
+              ]
+            }
+          }
+        }
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183"
+}

backend/Cargo.lock

@@ -696,9 +696,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
 
 [[package]]
 name = "async-trait"
-version = "0.1.82"
+version = "0.1.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
+checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1056,9 +1056,9 @@ dependencies = [
 
 [[package]]
 name = "axum"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
+checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -1083,7 +1083,7 @@ dependencies = [
  "serde_urlencoded",
  "sync_wrapper 1.0.1",
  "tokio",
- "tower 0.4.13",
+ "tower 0.5.1",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -1091,9 +1091,9 @@ dependencies = [
 
 [[package]]
 name = "axum-core"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3"
+checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00"
 dependencies = [
  "async-trait",
  "bytes",
@@ -1104,7 +1104,7 @@ dependencies = [
  "mime",
  "pin-project-lite",
  "rustversion",
- "sync_wrapper 0.1.2",
+ "sync_wrapper 1.0.1",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -1735,9 +1735,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.17"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac"
+checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1745,9 +1745,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.17"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73"
+checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1757,9 +1757,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.13"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
@@ -4670,9 +4670,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.158"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libgit2-sys"
@@ -4710,7 +4710,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
  "bitflags 2.6.0",
  "libc",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
 ]
 
 [[package]]
@@ -5442,7 +5442,7 @@ dependencies = [
  "md-5 0.10.6",
  "parking_lot",
  "percent-encoding",
- "quick-xml 0.36.1",
+ "quick-xml 0.36.2",
  "rand 0.8.5",
  "reqwest 0.12.7",
  "ring 0.17.8",
@@ -5672,7 +5672,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
  "smallvec",
  "windows-targets 0.52.6",
 ]
@@ -5991,9 +5991,9 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "polling"
@@ -6012,9 +6012,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce"
 
 [[package]]
 name = "postgres-native-tls"
@@ -6120,7 +6120,7 @@ version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
 dependencies = [
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -6374,9 +6374,9 @@ dependencies = [
 
 [[package]]
 name = "quick-xml"
-version = "0.36.1"
+version = "0.36.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc"
+checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe"
 dependencies = [
  "memchr",
  "serde",
@@ -6623,9 +6623,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.4"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
+checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0"
 dependencies = [
  "bitflags 2.6.0",
 ]
@@ -7415,9 +7415,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.11.1"
+version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf"
+checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -7755,9 +7755,9 @@ dependencies = [
 
 [[package]]
 name = "simdutf8"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
 [[package]]
 name = "simple_asn1"
@@ -9019,9 +9019,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.41"
+version = "0.4.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909"
+checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020"
 dependencies = [
  "filetime",
  "libc",
@@ -9061,18 +9061,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.63"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
+checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.63"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
+checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9433,7 +9433,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -9460,15 +9460,15 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.21"
+version = "0.22.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
 dependencies = [
  "indexmap 2.5.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.18",
+ "winnow 0.6.19",
 ]
 
 [[package]]
@@ -9497,7 +9497,6 @@ dependencies = [
  "tokio",
  "tower-layer",
  "tower-service",
- "tracing",
 ]
 
 [[package]]
@@ -9506,8 +9505,14 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f"
 dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper 0.1.2",
+ "tokio",
  "tower-layer",
  "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -10377,7 +10382,7 @@ version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
  "wasite",
  "web-sys",
 ]
@@ -10421,7 +10426,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "axum",
@@ -10463,7 +10468,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -10497,6 +10502,7 @@ dependencies = [
  "lazy_static",
  "magic-crypt",
  "mail-parser",
+ "matchit",
  "mime_guess",
  "native-tls",
  "object_store",
@@ -10547,7 +10553,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "base64 0.21.7",
  "chrono",
@@ -10565,7 +10571,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "chrono",
  "serde",
@@ -10578,7 +10584,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -10622,7 +10628,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "regex",
  "rsmq_async",
@@ -10637,7 +10643,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "bytes",
@@ -10658,7 +10664,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -10667,7 +10673,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10679,7 +10685,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -10691,7 +10697,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10703,7 +10709,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
@@ -10714,7 +10720,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
@@ -10725,7 +10731,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -10743,7 +10749,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -10760,7 +10766,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10772,7 +10778,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10790,7 +10796,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "getrandom 0.2.15",
@@ -10811,7 +10817,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -10821,7 +10827,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -10854,7 +10860,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "wasm-bindgen",
  "wasm-bindgen-test",
@@ -10864,7 +10870,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.397.4"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -11136,9 +11142,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.18"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
+checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587"
 dependencies = [
  "memchr",
 ]

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.397.4"
+version = "1.399.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -27,7 +27,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.397.4"
+version = "1.399.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -252,6 +252,7 @@ indexmap = { version = "2.2.5", features = ["serde"]}
 tokio-native-tls = "^0"
 openssl = "=0.10"
 mail-parser = "^0"
+matchit = "=0.7.3"
 
 datafusion = "39.0.0"
 object_store = { version = "0.10.0", features = ["aws", "azure"] }

backend/ee-repo-ref.txt

@@ -1 +1 @@
-534f50aae2f89281304b3c3ba943d18afc07fad4
+3d37b6c31155265d8d026ae9d6ced0b433078f87

backend/migrations/20240917172503_job_routing.down.sql

@@ -0,0 +1,7 @@
+-- Add down migration script here
+DROP TABLE http_trigger;
+DROP TYPE http_method;
+
+ALTER TABLE script DROP COLUMN has_preprocessor;
+
+DROP FUNCTION prevent_route_path_change();

backend/migrations/20240917172503_job_routing.up.sql

@@ -0,0 +1,61 @@
+-- Add up migration script here
+CREATE TYPE http_method AS ENUM ('get', 'post', 'put', 'delete', 'patch');
+
+CREATE TABLE http_trigger (
+  path VARCHAR(255) NOT NULL,
+  route_path VARCHAR(255) NOT NULL,
+  route_path_key VARCHAR(255) NOT NULL,
+  script_path VARCHAR(255) NOT NULL,
+  is_flow BOOLEAN NOT NULL,
+  workspace_id VARCHAR(50) NOT NULL,
+  edited_by VARCHAR(50) NOT NULL,
+  email VARCHAR(255) NOT NULL,
+  edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  extra_perms JSONB NOT NULL DEFAULT '{}',
+  is_async BOOLEAN NOT NULL DEFAULT FALSE,
+  requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
+  http_method http_method NOT NULL,
+  PRIMARY KEY (path, workspace_id)
+);
+
+
+GRANT SELECT, UPDATE ON http_trigger TO windmill_user;
+GRANT ALL ON http_trigger TO windmill_admin;
+
+ALTER TABLE http_trigger ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY see_folder_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
+USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
+CREATE POLICY see_folder_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
+USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+
+CREATE POLICY see_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
+CREATE POLICY see_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
+USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups_select ON http_trigger FOR SELECT TO windmill_user
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
+CREATE POLICY see_extra_perms_groups_update ON http_trigger FOR UPDATE TO windmill_user
+USING (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));
+
+CREATE OR REPLACE FUNCTION prevent_route_path_change()
+RETURNS TRIGGER AS $$
+BEGIN
+    IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN
+        RAISE EXCEPTION 'Modification of route_path is only allowed by admins';
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER check_route_path_change
+BEFORE UPDATE ON http_trigger
+FOR EACH ROW
+EXECUTE FUNCTION prevent_route_path_change();
+
+ALTER TABLE script ADD COLUMN has_preprocessor BOOLEAN;

backend/migrations/20240919200009_migrate_smtp_settings.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240919200009_migrate_smtp_settings.up.sql

@@ -0,0 +1,8 @@
+-- Add up migration script here
+INSERT INTO global_settings (name, value)
+SELECT 'smtp_settings', config
+FROM config
+WHERE name = 'server'
+ON CONFLICT (name) DO UPDATE SET value = excluded.value;
+
+DELETE FROM config WHERE name = 'server';

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -16,7 +16,13 @@ pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_bash_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing bash script".to_string()))
     }
@@ -26,7 +32,13 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_powershell_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing powershell script".to_string()))
     }
@@ -173,7 +185,8 @@ non_required="${5:-}"
                         oidx: None
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-go/src/lib.rs

@@ -42,6 +42,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -49,6 +50,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -241,7 +243,8 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
                         oidx: None
                     },
                 ],
-                no_main_func: Some(false)
+                no_main_func: Some(false),
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-graphql/src/lib.rs

@@ -15,7 +15,13 @@ pub fn parse_graphql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_graphql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -119,7 +125,8 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") {
                         oidx: None
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-php/src/lib.rs

@@ -82,6 +82,7 @@ pub fn parse_php_signature(
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -89,6 +90,7 @@ pub fn parse_php_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -159,7 +161,8 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f
                         oidx: None
                     }
                 ],
-                no_main_func: Some(false)
+                no_main_func: Some(false),
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-py/src/lib.rs

@@ -63,9 +63,17 @@ pub fn parse_python_signature(
 ) -> anyhow::Result<MainArgSignature> {
     let main_name = override_main.unwrap_or("main".to_string());
 
+    let has_preprocessor = !filter_non_main(code, "preprocessor").is_empty();
+
     let filtered_code = filter_non_main(code, &main_name);
     if filtered_code.is_empty() {
-        return Err(anyhow::anyhow!("No main function found".to_string(),));
+        return Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: vec![],
+            no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
+        });
     }
     let ast = Suite::parse(&filtered_code, "main.py")
         .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
@@ -119,6 +127,7 @@ pub fn parse_python_signature(
                 })
                 .collect(),
             no_main_func: Some(false),
+            has_preprocessor: Some(has_preprocessor),
         })
     } else {
         Ok(MainArgSignature {
@@ -126,6 +135,7 @@ pub fn parse_python_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
         })
     }
 }
@@ -325,6 +335,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
                     },
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -389,6 +400,7 @@ def main(test1: str,
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -448,6 +460,7 @@ def main(test1: str,
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -491,6 +504,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
                     }
                 ],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
             }
         );
 
@@ -521,6 +535,59 @@ def main(test1: DynSelect_foo): return
                     oidx: None
                 }],
                 no_main_func: Some(false),
+                has_preprocessor: Some(false)
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_6() -> anyhow::Result<()> {
+        let code = r#"
+
+import os
+
+def hello(): return
+
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code, None)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![],
+                no_main_func: Some(true),
+                has_preprocessor: Some(false)
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_sig_7() -> anyhow::Result<()> {
+        let code = r#"
+
+import os
+
+def preprocessor(): return
+
+def main(): return
+
+
+
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code, None)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![],
+                no_main_func: Some(false),
+                has_preprocessor: Some(true)
             }
         );
 

backend/parsers/windmill-parser-rust/src/lib.rs

@@ -29,6 +29,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args,
             no_main_func: Some(false),
+            has_preprocessor: None,
         })
     } else {
         Ok(MainArgSignature {
@@ -36,6 +37,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: None,
         })
     }
 }
@@ -455,10 +457,7 @@ fn main(
         assert_eq!(ret.args[2].typ, Typ::Str(None));
 
         assert_eq!(ret.args[3].name, "my_string_vec");
-        assert_eq!(
-            ret.args[3].otyp,
-            Some("Vec < String >".to_string())
-        );
+        assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string()));
         assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None))));
     }
 

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -20,7 +20,13 @@ pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_mysql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -30,7 +36,13 @@ pub fn parse_pgsql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_pg_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -40,7 +52,13 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_bigquery_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -50,7 +68,13 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_snowflake_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -60,7 +84,13 @@ pub fn parse_mssql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
     let parsed = parse_mssql_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
-        Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args,
+            no_main_func: None,
+            has_preprocessor: None,
+        })
     } else {
         Err(anyhow!("Error parsing sql".to_string()))
     }
@@ -570,7 +600,8 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
                         oidx: Some(2),
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -618,7 +649,8 @@ SELECT $2::TEXT;
                         oidx: Some(3),
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -744,7 +776,8 @@ SELECT ?, ?;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -791,7 +824,8 @@ SELECT :param2;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -830,7 +864,8 @@ SELECT @token;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -877,7 +912,8 @@ SELECT ?;
                         oidx: None,
                     }
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 
@@ -924,7 +960,8 @@ SELECT @p2;
                         oidx: None,
                     },
                 ],
-                no_main_func: None
+                no_main_func: None,
+                has_preprocessor: None
             }
         );
 

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -159,6 +159,14 @@ pub fn parse_deno_signature(
         })?
         .body;
 
+    let has_preprocessor = ast.iter().any(|x| match x {
+        ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
+            decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, .. }),
+            ..
+        })) => &sym.to_string() == "preprocessor",
+        _ => false,
+    });
+
     let main_name = main_override.unwrap_or("main".to_string());
     let params = ast.into_iter().find_map(|x| match x {
         ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
@@ -178,6 +186,7 @@ pub fn parse_deno_signature(
                 .map(|x| parse_param(x, &cm, skip_dflt, &mut c))
                 .collect::<anyhow::Result<Vec<Arg>>>()?,
             no_main_func: Some(false),
+            has_preprocessor: Some(has_preprocessor),
         };
         Ok(r)
     } else {
@@ -186,6 +195,7 @@ pub fn parse_deno_signature(
             star_kwargs: false,
             args: vec![],
             no_main_func: Some(true),
+            has_preprocessor: Some(has_preprocessor),
         })
     }
 }

backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh

@@ -10,35 +10,41 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \
 OUT_DIR="pkg-ts"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
 
 # sql languages, graphql and bash/powershell, since they all use regex
 OUT_DIR="pkg-regex"
 wasm-pack build --release --target web --out-dir $OUT_DIR \
 	--features "sql-parser,graphql-parser,bash-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
 
 # python
 OUT_DIR="pkg-py"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
 
 # go
 OUT_DIR="pkg-go"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
 
 # php
 OUT_DIR="pkg-php"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
 
 # rust
 OUT_DIR="pkg-rust"
 wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \
 	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
-sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
+
+# ansible
+OUT_DIR="pkg-yaml"
+wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \
+	-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
+sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json

backend/parsers/windmill-parser-wasm/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eou pipefail
 
-deno task wasmbuild --out ../../../cli/wasm/  -p windmill-parser-wasm
+deno task wasmbuild --out ../../../cli/wasm/  -p windmill-parser-wasm --all-features

backend/parsers/windmill-parser-wasm/publish-pkgs.sh

@@ -21,3 +21,6 @@ popd
 
 pushd "pkg-rust" && npm publish ${args}
 popd
+
+pushd "pkg-yaml" && npm publish ${args}
+popd

backend/parsers/windmill-parser-wasm/src/lib.rs

@@ -17,8 +17,12 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
 
 #[cfg(feature = "ts-parser")]
 #[wasm_bindgen]
-pub fn parse_deno(code: &str) -> String {
-    wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None))
+pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
+    wrap_sig(windmill_parser_ts::parse_deno_signature(
+        code,
+        false,
+        main_override,
+    ))
 }
 
 #[cfg(feature = "ts-parser")]
@@ -65,8 +69,11 @@ pub fn parse_go(code: &str) -> String {
 
 #[cfg(feature = "py-parser")]
 #[wasm_bindgen]
-pub fn parse_python(code: &str) -> String {
-    wrap_sig(windmill_parser_py::parse_python_signature(code, None))
+pub fn parse_python(code: &str, main_override: Option<String>) -> String {
+    wrap_sig(windmill_parser_py::parse_python_signature(
+        code,
+        main_override,
+    ))
 }
 
 #[cfg(feature = "sql-parser")]

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -139,7 +139,8 @@ export function main(test1?: string, test2: string = \"burkina\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -215,7 +216,8 @@ export function main(test2 = \"burkina\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -264,7 +266,8 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
                     oidx: None
                 }
             ],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -294,7 +297,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
                 has_default: false,
                 oidx: None
             }],
-            no_main_func: Some(false)
+            no_main_func: Some(false),
+            has_preprocessor: Some(false)
         }
     );
 
@@ -434,6 +438,7 @@ Write-Output 'Testing...'
                 }
             ],
             no_main_func: None,
+            has_preprocessor: None
         }
     );
 

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -13,6 +13,7 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
             star_kwargs: false,
             args: vec![],
             no_main_func: None,
+            has_preprocessor: None,
         });
     }
 
@@ -24,14 +25,19 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
                     if let Yaml::Hash(v) = &value {
                         for (key, arg) in v {
                             if let Yaml::String(arg_name) = key {
-                                args.push(Arg {
-                                    name: arg_name.to_string(),
-                                    otyp: None,
-                                    typ: parse_ansible_typ(&arg),
-                                    default: None,
-                                    has_default: false,
-                                    oidx: None,
-                                })
+                                // if not then it is a static resource or var, so skip
+                                // it will be fetched by the worker
+                                if let ArgTyp::Typ(typ) = parse_ansible_arg_typ(arg) {
+                                    let default = get_default_for_typ(&arg);
+                                    args.push(Arg {
+                                        name: arg_name.to_string(),
+                                        otyp: None,
+                                        typ,
+                                        has_default: default.is_some(),
+                                        default,
+                                        oidx: None,
+                                    })
+                                }
                             }
                         }
                     }
@@ -55,7 +61,57 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
             }
         }
     }
-    Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
+    Ok(MainArgSignature {
+        star_args: false,
+        star_kwargs: false,
+        args,
+        no_main_func: None,
+        has_preprocessor: None,
+    })
+}
+
+fn get_default_for_typ(arg: &Yaml) -> Option<serde_json::Value> {
+    if let Yaml::Hash(arg) = arg {
+        if let Some(def) = arg.get(&Yaml::String("default".to_string())) {
+            if let Some(Yaml::String(typ)) = arg.get(&Yaml::String("type".to_string())) {
+                if let Yaml::String(path) = def {
+                    if typ.as_str() == "windmill_resource" {
+                        return Some(serde_json::Value::String(format!("$res:{}", path)));
+                    }
+                }
+            }
+            return Some(yaml_to_json(def));
+        }
+    }
+    None
+}
+
+enum ArgTyp {
+    Typ(Typ),
+    StaticVar(String),
+    StaticResource(String),
+}
+
+fn parse_ansible_arg_typ(arg: &Yaml) -> ArgTyp {
+    if let Yaml::Hash(a) = arg {
+        if let Some(Yaml::String(typ)) = a.get(&Yaml::String("type".to_string())) {
+            if typ.as_str() == "windmill_variable" {
+                if let Some(Yaml::String(variable_path)) =
+                    a.get(&Yaml::String("variable".to_string()))
+                {
+                    return ArgTyp::StaticVar(variable_path.to_string());
+                }
+            }
+            if typ.as_str() == "windmill_resource" {
+                if let Some(Yaml::String(resource_path)) =
+                    a.get(&Yaml::String("resource".to_string()))
+                {
+                    return ArgTyp::StaticResource(resource_path.to_string());
+                }
+            }
+        }
+    }
+    ArgTyp::Typ(parse_ansible_typ(arg))
 }
 
 fn parse_ansible_typ(arg: &Yaml) -> Typ {
@@ -120,7 +176,19 @@ fn parse_ansible_typ(arg: &Yaml) -> Typ {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
+pub struct AnsiblePlaybookOptions {
+    // There are a lot more options
+    // TODO: Add the options as customers require them
+    // Add it here and then on the executors cmd_options
+    pub verbosity: Option<String>,
+    pub forks: Option<i64>,
+    pub timeout: Option<i64>,
+    pub flush_cache: Option<()>,
+    pub force_handlers: Option<()>,
+}
+
+#[derive(Debug, Clone)]
 pub struct FileResource {
     pub resource_path: String,
     pub target_path: String,
@@ -133,12 +201,15 @@ pub struct AnsibleInventory {
     resource_type: Option<String>,
     pub pinned_resource: Option<String>,
 }
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct AnsibleRequirements {
     pub python_reqs: Vec<String>,
     pub collections: Option<String>,
     pub file_resources: Vec<FileResource>,
     pub inventories: Vec<AnsibleInventory>,
+    pub vars: Vec<(String, String)>,
+    pub resources: Vec<(String, String)>,
+    pub options: AnsiblePlaybookOptions,
 }
 
 fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInventory>> {
@@ -173,7 +244,12 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInvento
                     .and_then(|v| v.as_str())
                     .map(|s| s.to_string());
 
-                ret.push(AnsibleInventory { default: windmill_path, name, resource_type, pinned_resource: pin });
+                ret.push(AnsibleInventory {
+                    default: windmill_path,
+                    name,
+                    resource_type,
+                    pinned_resource: pin,
+                });
             }
         }
         return Ok(ret);
@@ -192,11 +268,21 @@ pub fn parse_ansible_reqs(
         return Ok((logs, None, inner_content.to_string()));
     }
 
+    let opts = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
     let mut ret = AnsibleRequirements {
         python_reqs: vec![],
         collections: None,
         file_resources: vec![],
         inventories: vec![],
+        vars: vec![],
+        resources: vec![],
+        options: opts,
     };
 
     if let Yaml::Hash(doc) = &docs[0] {
@@ -230,10 +316,33 @@ pub fn parse_ansible_reqs(
                         ret.file_resources.append(&mut resources?);
                     }
                 }
-                Yaml::String(key) if key == "extra_vars" => {}
+                Yaml::String(key) if key == "extra_vars" => {
+                    if let Yaml::Hash(v) = &value {
+                        for (key, arg) in v {
+                            if let Yaml::String(arg_name) = key {
+                                match parse_ansible_arg_typ(arg) {
+                                    ArgTyp::StaticVar(p) => {
+                                        ret.vars
+                                            .push((arg_name.to_string(), format!("$var:{}", p)));
+                                    }
+                                    ArgTyp::StaticResource(p) => {
+                                        ret.resources
+                                            .push((arg_name.to_string(), format!("$res:{}", p)));
+                                    }
+                                    ArgTyp::Typ(_) => (),
+                                }
+                            }
+                        }
+                    }
+                }
                 Yaml::String(key) if key == "inventory" => {
                     ret.inventories = parse_inventories(value)?;
                 }
+                Yaml::String(key) if key == "options" => {
+                    if let Yaml::Array(opts) = &value {
+                        ret.options = parse_ansible_options(opts);
+                    }
+                }
                 Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
                 _ => (),
             }
@@ -248,10 +357,85 @@ pub fn parse_ansible_reqs(
     Ok((logs, Some(ret), out_str))
 }
 
+fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
+    let mut ret = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
+    for opt in opts {
+        if let Yaml::String(o) = opt {
+            match o.as_str() {
+                "flush_cache" => {
+                    ret.flush_cache = Some(());
+                }
+                "force_handlers" => {
+                    ret.force_handlers = Some(());
+                }
+                s if count_consecutive_vs(s) > 0 => {
+                    ret.verbosity = Some("v".repeat(count_consecutive_vs(s).min(6)));
+                }
+                _ => (),
+            }
+        }
+        if let Yaml::Hash(m) = opt {
+            if let Some((Yaml::String(name), value)) = m.iter().last() {
+                match name.as_str() {
+                    "forks" => {
+                        if let Yaml::Integer(count) = value {
+                            ret.forks = Some(*count);
+                        }
+                    }
+                    "timeout" => {
+                        if let Yaml::Integer(timeout) = value {
+                            ret.timeout = Some(*timeout);
+                        }
+                    }
+                    "verbosity" => {
+                        if let Yaml::String(verbosity) = value {
+                            let c = count_consecutive_vs(verbosity);
+                            if c > 0 && c <= 6 {
+                                ret.verbosity = Some("v".repeat(c.min(6)));
+                            }
+
+                        }
+                    }
+                    _ => ()
+
+                }
+            }
+
+
+        }
+    }
+
+    ret
+}
+
+fn count_consecutive_vs(s: &str) -> usize {
+    let mut max_count = 0;
+    let mut current_count = 0;
+
+    for c in s.chars() {
+        if c == 'v' {
+            current_count += 1;
+            if current_count == 6 {
+                return 6;  // Stop early if we reach 6
+            }
+        } else {
+            current_count = 0;  // Reset count if the character is not 'v'
+        }
+        max_count = max_count.max(current_count);
+    }
+
+    max_count
+}
+
 fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
     if let Yaml::Hash(f) = yaml {
-        if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string()))
-        {
+        if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) {
             let target_path = f
                 .get(&Yaml::String("target".to_string()))
                 .and_then(|x| x.as_str())
@@ -262,7 +446,37 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
                 ))?;
             return Ok(FileResource { resource_path: resource_path.clone(), target_path });
         }
-        return Err(anyhow!("File resource should have a `resource` field, linking to a text file resource"));
+        return Err(anyhow!(
+            "File resource should have a `resource` field, linking to a text file resource"
+        ));
     }
     return Err(anyhow!("Invalid file resource: Should be a dictionary."));
 }
+
+fn yaml_to_json(yaml: &Yaml) -> serde_json::Value {
+    match yaml {
+        Yaml::Array(arr) => {
+            let json_array: Vec<serde_json::Value> = arr.into_iter().map(yaml_to_json).collect();
+            serde_json::Value::Array(json_array)
+        }
+        Yaml::Hash(hash) => {
+            let json_object = hash
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = match k {
+                        Yaml::String(s) => s.clone(),
+                        _ => k.as_str().unwrap_or("").to_string(),
+                    };
+                    (key, yaml_to_json(v))
+                })
+                .collect();
+            serde_json::Value::Object(json_object)
+        }
+        Yaml::String(s) => serde_json::Value::String(s.to_string()),
+        Yaml::Integer(i) => serde_json::Value::Number(i.clone().into()),
+        Yaml::Real(r) => serde_json::Value::Number(r.parse().unwrap_or(0.into())),
+        Yaml::Boolean(b) => serde_json::Value::Bool(*b),
+        Yaml::Null => serde_json::Value::Null,
+        _ => serde_json::Value::Null,
+    }
+}

backend/parsers/windmill-parser/src/lib.rs

@@ -16,6 +16,7 @@ pub struct MainArgSignature {
     pub star_kwargs: bool,
     pub args: Vec<Arg>,
     pub no_main_func: Option<bool>,
+    pub has_preprocessor: Option<bool>,
 }
 
 #[derive(Serialize, Clone, Debug, PartialEq)]
@@ -153,7 +154,10 @@ mod test {
 
     #[test]
     fn test_mixed_delimiters() {
-        assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces");
+        assert_eq!(
+            to_snake_case("test-Case_with Spaces"),
+            "test_case_with_spaces"
+        );
     }
 
     #[test]

backend/src/main.rs

@@ -8,7 +8,10 @@
 
 use anyhow::Context;
 use git_version::git_version;
-use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store};
+use monitor::{
+    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
+    send_logs_to_object_store,
+};
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
@@ -35,7 +38,7 @@ use windmill_common::{
         JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
         OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -74,7 +77,7 @@ use crate::monitor::{
     reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
     reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
     reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
-    reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
+    reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
     reload_worker_config,
 };
 
@@ -563,9 +566,7 @@ Windmill Community Edition {GIT_VERSION}
                                         "notify_config_change" => {
                                             match n.payload() {
                                                 "server" if server_mode => {
-                                                    tracing::info!("Server config change detected: {}", n.payload());
-
-                                                    reload_server_config(&db).await;
+                                                    tracing::error!("Server config change detected but server config is obsolete: {}", n.payload());
                                                 },
                                                 a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
                                                     tracing::info!("Worker config change detected: {}", n.payload());
@@ -609,6 +610,12 @@ Windmill Community Edition {GIT_VERSION}
                                                         tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
                                                     }
                                                 }
+                                                SMTP_SETTING => {
+                                                    reload_smtp_config(&db).await;
+                                                },
+                                                TIMEOUT_WAIT_RESULT_SETTING => {
+                                                    reload_timeout_wait_result_setting(&db).await
+                                                },
                                                 RETENTION_PERIOD_SECS_SETTING => {
                                                     reload_retention_period_setting(&db).await
                                                 },

backend/src/monitor.rs

@@ -22,6 +22,7 @@ use tokio::{
 #[cfg(feature = "embedding")]
 use windmill_api::embeddings::update_embeddings_db;
 use windmill_api::{
+    jobs::TIMEOUT_WAIT_RESULT,
     oauth2_ee::{build_oauth_clients, OAuthClient},
     DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
 };
@@ -40,17 +41,17 @@ use windmill_common::{
         KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
         PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     jobs::QueuedJob,
     oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
-    server::load_server_config,
+    server::load_smtp_config,
     tracing_init::JSON_FMT,
     users::truncate_token,
     utils::{now_from_db, rd_string, report_critical_error, Mode},
     worker::{
         load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
-        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
+        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG,
         WORKER_GROUP,
     },
     BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
@@ -172,8 +173,9 @@ pub async fn initial_load(
         reload_s3_cache_setting(&db).await;
     }
 
+    reload_smtp_config(&db).await;
+
     if server_mode {
-        reload_server_config(&db).await;
         reload_retention_period_setting(&db).await;
         reload_request_size(&db).await;
         reload_saml_metadata_setting(&db).await;
@@ -698,6 +700,15 @@ pub async fn reload_scim_token_setting(db: &DB) {
         .await;
 }
 
+pub async fn reload_timeout_wait_result_setting(db: &DB) {
+    reload_option_setting_with_tracing(
+        db,
+        TIMEOUT_WAIT_RESULT_SETTING,
+        "TIMEOUT_WAIT_RESULT",
+        TIMEOUT_WAIT_RESULT.clone(),
+    )
+    .await;
+}
 pub async fn reload_saml_metadata_setting(db: &DB) {
     reload_option_setting_with_tracing(
         db,
@@ -1129,14 +1140,14 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
     }
 }
 
-pub async fn reload_server_config(db: &Pool<Postgres>) {
-    let config = load_server_config(&db).await;
-    if let Err(e) = config {
-        tracing::error!("Error reloading server config: {:?}", e)
+pub async fn reload_smtp_config(db: &Pool<Postgres>) {
+    let smtp_config = load_smtp_config(&db).await;
+    if let Err(e) = smtp_config {
+        tracing::error!("Error reloading smtp config: {:?}", e)
     } else {
-        let mut wc = SERVER_CONFIG.write().await;
-        tracing::info!("Reloading server config...");
-        *wc = config.unwrap()
+        let mut wc = SMTP_CONFIG.write().await;
+        tracing::info!("Reloading smtp config...");
+        *wc = smtp_config.unwrap()
     }
 }
 

backend/tests/worker.rs

@@ -1912,6 +1912,109 @@ def main():
     assert_eq!(result, serde_json::json!("test-workspace"));
 }
 
+#[sqlx::test(fixtures("base"))]
+async fn test_bun_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Bun,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_deno_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Deno,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+from datetime import datetime
+def main(a: datetime, b: bytes):
+    return (isinstance(a, datetime), isinstance(b, bytes))
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Python3,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .arg("b", json!("dGVzdA=="))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!([true, true]));
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_empty_loop_1(db: Pool<Postgres>) {
     initialize_tracing().await;
@@ -3294,6 +3397,7 @@ async fn run_deployed_relative_imports(
                 visible_to_runner_only: None,
                 no_main_func: None,
                 codebase: None,
+                has_preprocessor: None,
             },
         )
         .await
@@ -3324,6 +3428,7 @@ async fn run_deployed_relative_imports(
                 dedicated_worker: None,
                 language,
                 priority: None,
+                apply_preprocessor: false,
             })
             .push(&db2)
             .await;

backend/windmill-api/Cargo.toml

@@ -94,6 +94,7 @@ object_store = { workspace = true, optional = true}
 openidconnect = { workspace = true, optional = true}
 url = { workspace = true, optional = true}
 jsonwebtoken = { workspace = true }
+matchit.workspace = true
 
 pin-project.workspace = true
 crc.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.397.4
+  version: 1.399.0
   title: Windmill API
 
   contact:
@@ -742,6 +742,35 @@ paths:
               schema:
                 type: string
 
+  /settings/test_critical_channels:
+    post:
+      summary: test critical channels
+      operationId: testCriticalChannels
+      tags:
+        - setting
+      requestBody:
+        description: test critical channel payload
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                type: object
+                properties:
+                  email:
+                    type: string
+                  slack_channel:
+                    type: string
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain:
+              schema:
+                type: string
+
+
   /settings/test_license_key:
     post:
       summary: test license key
@@ -4182,6 +4211,11 @@ paths:
           in: query
           schema:
             type: integer
+        - name: skip_preprocessor
+          description: skip the preprocessor
+          in: query
+          schema:
+            type: boolean
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/CacheTtl"
@@ -5328,6 +5362,11 @@ paths:
           in: query
           schema:
             type: integer
+        - name: skip_preprocessor
+          description: skip the preprocessor
+          in: query
+          schema:
+            type: boolean
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/NewJobId"
@@ -5436,6 +5475,11 @@ paths:
           in: query
           schema:
             type: integer
+        - name: skip_preprocessor
+          description: skip the preprocessor
+          in: query
+          schema:
+            type: boolean
         - $ref: "#/components/parameters/ParentJob"
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/CacheTtl"
@@ -6822,6 +6866,190 @@ paths:
         "201":
           description: default error handler set
 
+
+  /w/{workspace}/http_triggers/create:
+    post:
+      summary: create http trigger
+      operationId: createHttpTrigger
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: new http trigger
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/NewHttpTrigger"
+      responses:
+        "201":
+          description: http trigger created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/http_triggers/update/{path}:
+    post:
+      summary: update http trigger
+      operationId: updateHttpTrigger
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      requestBody:
+        description: updated trigger
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/EditHttpTrigger"
+      responses:
+        "200":
+          description: http trigger updated
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/http_triggers/delete/{path}:
+    delete:
+      summary: delete http trigger
+      operationId: deleteHttpTrigger
+      tags:
+        -  http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: http trigger deleted
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/http_triggers/get/{path}:
+    get:
+      summary: get http trigger
+      operationId: getHttpTrigger
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: http trigger deleted
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/HttpTrigger"
+
+  
+  /w/{workspace}/http_triggers/list:
+    get:
+      summary: list http triggers
+      operationId: listHttpTriggers
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+          required: true
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+        - name: path
+          description: filter by path
+          in: query
+          schema:
+            type: string
+        - name: is_flow
+          in: query
+          schema:
+            type: boolean
+        - name: path_start
+          in: query
+          schema:
+            type: string
+      responses:
+        "200":
+          description: http trigger list
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/HttpTrigger"
+
+  
+  /w/{workspace}/http_triggers/exists/{path}:
+    get:
+      summary: does http trigger exists
+      operationId: existsHttpTrigger
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: http trigger exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
+  /w/{workspace}/http_triggers/route_exists:
+    post:
+      summary: does route exists
+      operationId: existsRoute
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: route exists request
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                route_path:
+                  type: string
+                http_method:
+                  type: string
+                  enum: ["get", "post", "put", "delete", "patch"]
+              required:
+                - kind
+                - route_path
+                - http_method
+      responses:
+        "200":
+          description: route exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
+  /w/{workspace}/http_triggers/used:
+    get:
+      summary: whether http triggers are used
+      operationId: used
+      tags:
+        - http_trigger
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: whether http triggers are used
+          content:
+            application/json:
+              schema:
+                type: boolean
+
   /groups/list:
     get:
       summary: list instance groups
@@ -7648,6 +7876,7 @@ paths:
                 folder,
                 app,
                 raw_app,
+                http_trigger,
               ]
       responses:
         "200":
@@ -7684,6 +7913,7 @@ paths:
                 folder,
                 app,
                 raw_app,
+                http_trigger,
               ]
       requestBody:
         description: acl to add
@@ -7731,6 +7961,7 @@ paths:
                 folder,
                 app,
                 raw_app,
+                http_trigger,
               ]
       requestBody:
         description: acl to add
@@ -9421,6 +9652,8 @@ components:
           type: boolean
         codebase:
           type: string
+        has_preprocessor:
+          type: boolean
 
       required:
         - hash
@@ -9438,6 +9671,7 @@ components:
         - kind
         - starred
         - no_main_func
+        - has_preprocessor
 
 
     NewScript:
@@ -9519,6 +9753,8 @@ components:
           type: boolean
         codebase:
           type: string
+        has_preprocessor:
+          type: boolean
       required:
         - path
         - summary
@@ -10319,6 +10555,9 @@ components:
         no_main_func:
           type: boolean
           nullable: true
+        has_preprocessor:
+          type: boolean
+          nullable: true
       required:
         - star_args
         - start_kwargs
@@ -10326,6 +10565,7 @@ components:
         - type
         - error
         - no_main_func
+        - has_preprocessor
 
     Preview:
       type: object
@@ -10715,6 +10955,122 @@ components:
         - is_flow
         - args
 
+    HttpTrigger:
+      type: object
+      properties:
+        path:
+          type: string
+        edited_by:
+          type: string
+        edited_at:
+          type: string
+          format: date-time
+        script_path:
+          type: string
+        route_path:
+          type: string
+        is_flow:
+          type: boolean
+        extra_perms:
+          type: object
+          additionalProperties:
+            type: boolean
+        email:
+          type: string
+        workspace_id:
+          type: string
+        http_method:
+          type: string
+          enum:
+            - get
+            - post
+            - put
+            - delete
+            - patch
+        is_async:
+          type: boolean
+        requires_auth:
+          type: boolean
+        
+      required:
+        - path
+        - edited_by
+        - edited_at
+        - script_path
+        - route_path
+        - extra_perms
+        - is_flow
+        - email
+        - workspace_id
+        - is_async
+        - requires_auth
+        - http_method
+
+    NewHttpTrigger:
+      type: object
+      properties:
+        path:
+          type: string
+        script_path:
+          type: string
+        route_path:
+          type: string
+        is_flow:
+          type: boolean
+        http_method:
+          type: string
+          enum:
+            - get
+            - post
+            - put
+            - delete
+            - patch
+        is_async:
+          type: boolean
+        requires_auth:
+          type: boolean
+        
+      required:
+        - path
+        - script_path
+        - route_path
+        - is_flow
+        - is_async
+        - requires_auth
+        - http_method
+
+    EditHttpTrigger:
+      type: object
+      properties:
+        path:
+          type: string
+        script_path:
+          type: string
+        route_path:
+          type: string
+        is_flow:
+          type: boolean
+        http_method:
+          type: string
+          enum:
+            - get
+            - post
+            - put
+            - delete
+            - patch
+        is_async:
+          type: boolean
+        requires_auth:
+          type: boolean
+      required:
+        - path
+        - script_path
+        - is_flow
+        - kind
+        - is_async
+        - requires_auth
+        - http_method
+
     Group:
       type: object
       properties:

backend/windmill-api/src/concurrency_groups.rs

@@ -144,7 +144,7 @@ async fn get_concurrent_intervals(
     Query(iq): Query<ExtendedJobsParams>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> JsonResult<ExtendedJobs> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     if lq.success.is_some() && lq.running.is_some_and(|x| x) {
         return Err(error::Error::BadRequest(

backend/windmill-api/src/flows.rs

@@ -1258,6 +1258,7 @@ mod tests {
                 delete_after_use: None,
                 continue_on_error: None,
             })),
+            preprocessor_module: None,
             same_worker: false,
             concurrent_limit: None,
             concurrency_time_window_s: None,
@@ -1330,7 +1331,8 @@ mod tests {
                 "expr": "previous.isEmpty()",
                 "skip_if_stopped": false
             }
-          }
+          },
+          "preprocessor_module": Option::<String>::None
         });
         assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));
     }

backend/windmill-api/src/granular_acls.rs

@@ -23,8 +23,17 @@ use windmill_common::{
     utils::{not_found_if_none, StripPath},
 };
 
-const KINDS: [&str; 9] = [
-    "script", "group_", "resource", "schedule", "variable", "flow", "folder", "app", "raw_app",
+const KINDS: [&str; 10] = [
+    "script",
+    "group_",
+    "resource",
+    "schedule",
+    "variable",
+    "flow",
+    "folder",
+    "app",
+    "raw_app",
+    "http_trigger",
 ];
 
 pub fn workspaced_service() -> Router {

backend/windmill-api/src/http_triggers.rs

@@ -0,0 +1,651 @@
+use axum::{
+    extract::{Path, Query},
+    response::IntoResponse,
+    routing::{delete, get, post},
+    Extension, Json, Router,
+};
+use http::{HeaderMap, StatusCode};
+use serde::{Deserialize, Serialize};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::prelude::FromRow;
+use std::collections::HashMap;
+use tower_http::cors::CorsLayer;
+use windmill_audit::{audit_ee::audit_log, ActionKind};
+use windmill_common::{
+    auth::fetch_authed_from_permissioned_as,
+    db::UserDB,
+    error::{self, JsonResult},
+    users::username_to_permissioned_as,
+    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    worker::{to_raw_value, CLOUD_HOSTED},
+};
+use windmill_queue::PushArgsOwned;
+
+use crate::{
+    db::{ApiAuthed, DB},
+    jobs::{
+        run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal,
+        run_wait_result_script_by_path_internal, RunJobQuery,
+    },
+    users::OptAuthed,
+};
+
+lazy_static::lazy_static! {
+    static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap();
+}
+
+pub fn routes_global_service() -> Router {
+    let cors = CorsLayer::new()
+        .allow_methods([
+            http::Method::GET,
+            http::Method::POST,
+            http::Method::DELETE,
+            http::Method::PUT,
+            http::Method::PATCH,
+        ])
+        .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
+        .allow_origin(tower_http::cors::Any);
+    Router::new()
+        .route(
+            "/*path",
+            get(route_job)
+                .post(route_job)
+                .delete(route_job)
+                .put(route_job)
+                .patch(route_job)
+                .head(|| async { "" }),
+        )
+        .layer(cors)
+}
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/create", post(create_trigger))
+        .route("/list", get(list_triggers))
+        .route("/get/*path", get(get_trigger))
+        .route("/update/*path", post(update_trigger))
+        .route("/delete/*path", delete(delete_trigger))
+        .route("/exists/*path", get(exists_trigger))
+        .route("/used", get(used))
+        .route("/route_exists", post(exists_route))
+}
+
+#[derive(Serialize, Deserialize, sqlx::Type)]
+#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
+#[serde(rename_all = "lowercase")]
+enum HttpMethod {
+    Get,
+    Post,
+    Put,
+    Delete,
+    Patch,
+}
+
+impl From<HttpMethod> for http::Method {
+    fn from(method: HttpMethod) -> Self {
+        match method {
+            HttpMethod::Get => http::Method::GET,
+            HttpMethod::Post => http::Method::POST,
+            HttpMethod::Put => http::Method::PUT,
+            HttpMethod::Delete => http::Method::DELETE,
+            HttpMethod::Patch => http::Method::PATCH,
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct NewTrigger {
+    path: String,
+    route_path: String,
+    script_path: String,
+    is_flow: bool,
+    is_async: bool,
+    requires_auth: bool,
+    http_method: HttpMethod,
+}
+
+#[derive(FromRow, Serialize)]
+struct Trigger {
+    workspace_id: String,
+    path: String,
+    route_path: String,
+    route_path_key: String,
+    script_path: String,
+    is_flow: bool,
+    edited_by: String,
+    email: String,
+    edited_at: chrono::DateTime<chrono::Utc>,
+    extra_perms: serde_json::Value,
+    is_async: bool,
+    requires_auth: bool,
+    http_method: HttpMethod,
+}
+
+#[derive(Deserialize)]
+struct EditTrigger {
+    path: String,
+    route_path: Option<String>,
+    script_path: String,
+    is_flow: bool,
+    is_async: bool,
+    requires_auth: bool,
+    http_method: HttpMethod,
+}
+
+#[derive(Deserialize)]
+pub struct ListTriggerQuery {
+    pub page: Option<usize>,
+    pub per_page: Option<usize>,
+    pub path: Option<String>,
+    pub is_flow: Option<bool>,
+    pub path_start: Option<String>,
+}
+
+async fn list_triggers(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(lst): Query<ListTriggerQuery>,
+) -> error::JsonResult<Vec<Trigger>> {
+    let mut tx = user_db.begin(&authed).await?;
+    let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page });
+    let mut sqlb = SqlBuilder::select_from("http_trigger")
+        .field("*")
+        .order_by("edited_at", true)
+        .and_where("workspace_id = ?".bind(&w_id))
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+    if let Some(path) = lst.path {
+        sqlb.and_where_eq("script_path", "?".bind(&path));
+    }
+    if let Some(is_flow) = lst.is_flow {
+        sqlb.and_where_eq("is_flow", "?".bind(&is_flow));
+    }
+    if let Some(path_start) = &lst.path_start {
+        sqlb.and_where_like_left("path", path_start);
+    }
+    let sql = sqlb
+        .sql()
+        .map_err(|e| error::Error::InternalErr(e.to_string()))?;
+    let rows = sqlx::query_as::<_, Trigger>(&sql)
+        .fetch_all(&mut *tx)
+        .await?;
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn get_trigger(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> error::JsonResult<Trigger> {
+    let mut tx = user_db.begin(&authed).await?;
+    let path = path.to_path();
+    let trigger = sqlx::query_as!(
+        Trigger,
+        r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth
+            FROM http_trigger
+            WHERE workspace_id = $1 AND path = $2"#,
+        w_id,
+        path,
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+    tx.commit().await?;
+
+    let trigger = not_found_if_none(trigger, "Trigger", path)?;
+
+    Ok(Json(trigger))
+}
+
+async fn create_trigger(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Json(ct): Json<NewTrigger>,
+) -> error::Result<(StatusCode, String)> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    let route_path_key = ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_str(), ":key");
+
+    let mut tx = user_db.begin(&authed).await?;
+    sqlx::query!(
+        "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
+        w_id,
+        ct.path,
+        ct.route_path,
+        &route_path_key,
+        ct.script_path,
+        ct.is_flow,
+        ct.is_async,
+        ct.requires_auth,
+        ct.http_method as HttpMethod,
+        &authed.username,
+        &authed.email
+    )
+    .execute(&mut *tx).await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "http_triggers.create",
+        ActionKind::Create,
+        &w_id,
+        Some(ct.path.as_str()),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok((StatusCode::CREATED, format!("{}", ct.path)))
+}
+
+async fn update_trigger(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(ct): Json<EditTrigger>,
+) -> error::Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    if authed.is_admin {
+        if ct.route_path.is_none() {
+            return Err(error::Error::BadRequest(
+                "route_path is required".to_string(),
+            ));
+        }
+
+        let route_path_key =
+            ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_ref().unwrap().as_str(), ":key");
+
+        sqlx::query!(
+            "UPDATE http_trigger 
+                SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() 
+                WHERE workspace_id = $11 AND path = $12",
+            ct.route_path,
+            &route_path_key,
+            ct.script_path,
+            ct.path,
+            ct.is_flow,
+            ct.http_method as HttpMethod,
+            &authed.username,
+            &authed.email,
+            ct.is_async,
+            ct.requires_auth,
+            w_id,
+            path,
+        )
+        .execute(&mut *tx).await?;
+    } else {
+        sqlx::query!(
+            "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() 
+                WHERE workspace_id = $9 AND path = $10",
+            ct.script_path,
+            ct.path,
+            ct.is_flow,
+            ct.http_method as HttpMethod,
+            &authed.username,
+            &authed.email,
+            ct.is_async,
+            ct.requires_auth,
+            w_id,
+            path,
+        )
+        .execute(&mut *tx).await?;
+    }
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "http_triggers.update",
+        ActionKind::Create,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(path.to_string())
+}
+
+async fn delete_trigger(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> error::Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+    sqlx::query!(
+        "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
+        w_id,
+        path,
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "http_triggers.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(format!("HTTP trigger {path} deleted"))
+}
+
+async fn used(Extension(db): Extension<DB>, Path(w_id): Path<String>) -> JsonResult<bool> {
+    let used = sqlx::query_scalar!(
+        r#"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)"#,
+        w_id,
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+    Ok(Json(used))
+}
+
+async fn exists_trigger(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id,
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+    Ok(Json(exists))
+}
+
+#[derive(Deserialize)]
+struct RouteExists {
+    route_path: String,
+    http_method: HttpMethod,
+}
+
+async fn exists_route(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(RouteExists { route_path, http_method }): Json<RouteExists>,
+) -> JsonResult<bool> {
+    let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key");
+    let exists = if *CLOUD_HOSTED {
+        sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)",
+                    &route_path_key,
+                    w_id,
+                    http_method as HttpMethod
+                )
+                .fetch_one(&db)
+                .await?
+                .unwrap_or(false)
+    } else {
+        sqlx::query_scalar!(
+            "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)",
+            &route_path_key,
+            http_method as HttpMethod
+        )
+        .fetch_one(&db)
+        .await?
+        .unwrap_or(false)
+    };
+    Ok(Json(exists))
+}
+
+struct TriggerRoute {
+    path: String,
+    script_path: String,
+    is_flow: bool,
+    route_path: String,
+    workspace_id: String,
+    is_async: bool,
+    requires_auth: bool,
+    edited_by: String,
+    email: String,
+    http_method: HttpMethod,
+}
+
+async fn fetch_api_authed(
+    username: String,
+    email: String,
+    w_id: &str,
+    db: &DB,
+    username_override: String,
+) -> error::Result<ApiAuthed> {
+    let permissioned_as = username_to_permissioned_as(username.as_str());
+    let authed =
+        fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
+    Ok(ApiAuthed {
+        username: username,
+        email: email,
+        is_admin: authed.is_admin,
+        is_operator: authed.is_operator,
+        groups: authed.groups,
+        folders: authed.folders,
+        scopes: authed.scopes,
+        username_override: Some(username_override),
+    })
+}
+
+async fn get_http_route_trigger(
+    route_path: &str,
+    opt_authed: Option<ApiAuthed>,
+    db: &DB,
+    user_db: UserDB,
+) -> error::Result<(TriggerRoute, String, HashMap<String, String>, ApiAuthed)> {
+    let (mut triggers, route_path) = if *CLOUD_HOSTED {
+        let mut splitted = route_path.split("/");
+        let w_id = splitted.next().ok_or_else(|| {
+            error::Error::BadRequest("Missing workspace id in route path".to_string())
+        })?;
+        let route_path = StripPath(splitted.collect::<Vec<_>>().join("/"));
+        let triggers = sqlx::query_as!(
+            TriggerRoute,
+            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#,
+            w_id
+        )
+        .fetch_all(db)
+        .await?;
+        (triggers, route_path)
+    } else {
+        let triggers = sqlx::query_as!(
+            TriggerRoute,
+            r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#,
+        )
+        .fetch_all(db)
+        .await?;
+        (triggers, StripPath(route_path.to_string()))
+    };
+
+    let mut router = matchit::Router::new();
+
+    for (idx, trigger) in triggers.iter().enumerate() {
+        let route_path = trigger.route_path.clone();
+        router.insert(route_path.as_str(), idx).unwrap_or_else(|e| {
+            tracing::warn!(
+                "Failed to consider http trigger route {}: {:?}",
+                route_path,
+                e,
+            );
+        });
+    }
+
+    let trigger_idx = router.at(route_path.0.as_str()).ok();
+
+    let matchit::Match { value: trigger_idx, params } =
+        not_found_if_none(trigger_idx, "Trigger", route_path.0.as_str())?;
+
+    let trigger = triggers.remove(trigger_idx.to_owned());
+
+    let params: HashMap<String, String> = params
+        .iter()
+        .map(|(k, v)| (k.to_string(), v.to_string()))
+        .collect();
+
+    let username_override = if trigger.requires_auth {
+        if let Some(authed) = opt_authed {
+            // check that the user has access to the trigger
+            let mut tx = user_db.begin(&authed).await?;
+            let exists = sqlx::query_scalar!(
+                "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
+                trigger.workspace_id,
+                trigger.path
+            )
+            .fetch_one(&mut *tx)
+            .await?
+            .unwrap_or(false);
+            tx.commit().await?;
+            if exists {
+                Some(authed.display_username().to_owned())
+            } else {
+                return Err(error::Error::NotAuthorized("Unauthorized".to_string()));
+            }
+        } else {
+            return Err(error::Error::NotAuthorized(
+                "Requires authentication".to_string(),
+            ));
+        }
+    } else {
+        None
+    };
+
+    let authed = fetch_api_authed(
+        trigger.edited_by.clone(),
+        trigger.email.clone(),
+        &trigger.workspace_id,
+        &db,
+        username_override.unwrap_or("anonymous".to_string()),
+    )
+    .await?;
+
+    Ok((trigger, route_path.0, params, authed))
+}
+
+async fn route_job(
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
+    Path(route_path): Path<StripPath>,
+    OptAuthed(opt_authed): OptAuthed,
+    Query(query): Query<HashMap<String, String>>,
+    method: http::Method,
+    headers: HeaderMap,
+    mut args: PushArgsOwned,
+) -> impl IntoResponse {
+    let route_path = route_path.to_path();
+    let (trigger, called_path, params, authed) =
+        match get_http_route_trigger(route_path, opt_authed, &db, user_db.clone()).await {
+            Ok(trigger) => trigger,
+            Err(e) => return e.into_response(),
+        };
+    let headers = headers
+        .iter()
+        .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
+        .collect::<HashMap<String, String>>();
+    let extra = args.extra.get_or_insert_with(HashMap::new);
+    extra.insert(
+        "wm_trigger".to_string(),
+        to_raw_value(&serde_json::json!({
+            "kind": "http",
+            "http": {
+                "route": trigger.route_path,
+                "path": called_path,
+                "method": method.to_string().to_lowercase(),
+                "params": params,
+                "query": query,
+                "headers": headers
+            },
+        })),
+    );
+    let http_method = http::Method::from(trigger.http_method);
+
+    if http_method != method {
+        return error::Error::BadRequest("Invalid HTTP method".to_string()).into_response();
+    }
+
+    let label_prefix = Some(format!(
+        "http-{}-{}-",
+        http_method.as_str().to_lowercase(),
+        trigger.route_path
+    ));
+
+    let run_query = RunJobQuery::default();
+
+    if trigger.is_flow {
+        if trigger.is_async {
+            run_flow_by_path_inner(
+                authed,
+                db,
+                user_db,
+                rsmq,
+                trigger.workspace_id.clone(),
+                StripPath(trigger.script_path.to_owned()),
+                run_query,
+                args,
+                label_prefix,
+            )
+            .await
+            .into_response()
+        } else {
+            run_wait_result_flow_by_path_internal(
+                db,
+                run_query,
+                StripPath(trigger.script_path.to_owned()),
+                authed,
+                rsmq,
+                user_db,
+                args,
+                trigger.workspace_id.clone(),
+                label_prefix,
+            )
+            .await
+            .into_response()
+        }
+    } else {
+        if trigger.is_async {
+            run_script_by_path_inner(
+                authed,
+                db,
+                user_db,
+                rsmq,
+                trigger.workspace_id.clone(),
+                StripPath(trigger.script_path.to_owned()),
+                run_query,
+                args,
+                label_prefix,
+            )
+            .await
+            .into_response()
+        } else {
+            run_wait_result_script_by_path_internal(
+                db,
+                run_query,
+                StripPath(trigger.script_path.to_owned()),
+                authed,
+                rsmq,
+                user_db,
+                trigger.workspace_id.clone(),
+                args,
+                label_prefix,
+            )
+            .await
+            .into_response()
+        }
+    }
+}

backend/windmill-api/src/jobs.rs

@@ -33,6 +33,7 @@ use crate::add_webhook_allowed_origin;
 use crate::concurrency_groups::join_concurrency_key;
 use crate::db::ApiAuthed;
 
+use crate::users::get_scope_tags;
 use crate::utils::content_plain;
 use crate::{
     db::DB,
@@ -58,7 +59,7 @@ use tower_http::cors::{Any, CorsLayer};
 use urlencoding::encode;
 use windmill_audit::audit_ee::{audit_log, AuditAuthor};
 use windmill_audit::ActionKind;
-use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, SERVER_CONFIG};
+use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE};
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error},
@@ -556,9 +557,10 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio
     Option<i16>,
     Option<bool>,
     Option<i32>,
+    Option<bool>,
 )> {
     let script = sqlx::query!(
-        "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2",
+        "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2",
         hash,
         w_id
     )
@@ -583,6 +585,7 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio
         script.priority,
         script.delete_after_use,
         script.timeout,
+        script.has_preprocessor,
     ))
 }
 
@@ -1058,6 +1061,7 @@ pub struct RunJobQuery {
     pub tag: Option<String>,
     pub timeout: Option<i32>,
     pub cache_ttl: Option<i32>,
+    pub skip_preprocessor: Option<bool>,
 }
 
 impl RunJobQuery {
@@ -1245,13 +1249,18 @@ pub fn list_queue_jobs_query(
     lq: &ListQueueQuery,
     fields: &[&str],
     join_outstanding_wait_times: bool,
+    tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
-    let sqlb = SqlBuilder::select_from("queue")
+    let mut sqlb = SqlBuilder::select_from("queue")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .limit(1000)
         .clone();
 
+    if let Some(tags) = tags {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times)
 }
 
@@ -1307,6 +1316,7 @@ async fn list_queue_jobs(
             "workspace_id",
         ],
         false,
+        get_scope_tags(&authed),
     )
     .sql()?;
     let mut tx = user_db.begin(&authed).await?;
@@ -1495,6 +1505,10 @@ async fn list_filtered_uuids(
 
     sqlb.and_where_is_null("schedule_path");
 
+    if let Some(tags) = get_scope_tags(&authed) {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     sqlb = filter_list_queue_query(sqlb, &lq, w_id.as_str(), false);
 
     let sql = sqlb.query()?;
@@ -1554,7 +1568,7 @@ async fn list_jobs(
     Query(lq): Query<ListCompletedQuery>,
     Extension(_api_list_jobs_query_duration): Extension<Option<Histo>>,
 ) -> error::JsonResult<Vec<Job>> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
@@ -1572,6 +1586,7 @@ async fn list_jobs(
             &ListCompletedQuery { order_desc: Some(true), ..lqc },
             UnifiedJob::completed_job_fields(),
             true,
+            get_scope_tags(&authed),
         ))
     } else {
         None
@@ -1587,6 +1602,7 @@ async fn list_jobs(
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
             UnifiedJob::queued_job_fields(),
             true,
+            get_scope_tags(&authed),
         );
 
         if let Some(sqlc) = sqlc {
@@ -1637,7 +1653,7 @@ pub async fn resume_suspended_flow_as_owner(
     Path((_w_id, flow_id)): Path<(String, Uuid)>,
     QueryOrBody(value): QueryOrBody<serde_json::Value>,
 ) -> error::Result<StatusCode> {
-    check_scopes(&authed, || format!("resumeflow"))?;
+    check_scopes(&authed, || format!("jobs:resumeflow"))?;
     let value = value.unwrap_or(serde_json::Value::Null);
     let mut tx = db.begin().await?;
 
@@ -2784,14 +2800,18 @@ pub async fn run_flow_by_path_inner(
 
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
-    let (tag, dedicated_worker) = sqlx::query!(
-        "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2",
+    let (tag, dedicated_worker, has_preprocessor) = sqlx::query!(
+        "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor 
+        FROM flow 
+        LEFT JOIN flow_version
+            ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
+        WHERE flow.path = $1 and flow.workspace_id = $2",
         flow_path,
         w_id
     )
     .fetch_optional(&mut tx)
     .await?
-    .map(|x| (x.tag, x.dedicated_worker))
+    .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor))
     .ok_or_else(|| {
         Error::NotFound(format!(
             "flow not found at path {flow_path} in workspace {w_id}"
@@ -2807,7 +2827,12 @@ pub async fn run_flow_by_path_inner(
         &db,
         tx,
         &w_id,
-        JobPayload::Flow { path: flow_path.to_string(), dedicated_worker },
+        JobPayload::Flow {
+            path: flow_path.to_string(),
+            dedicated_worker,
+            apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
+                && has_preprocessor.unwrap_or(false),
+        },
         PushArgs { args: &args.args, extra: args.extra },
         &label_prefix
             .map(|x| x + authed.display_username())
@@ -2973,7 +2998,7 @@ pub async fn run_script_by_path_inner(
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
     let (job_payload, tag, _delete_after_use, timeout) =
-        script_path_to_payload(script_path, &mut tx, &w_id).await?;
+        script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
     let scheduled_for = run_query.get_scheduled_for(&db).await?;
 
     let tag = run_query.tag.clone().or(tag);
@@ -3049,7 +3074,15 @@ pub async fn run_workflow_as_code(
             None,
             run_query.timeout,
         ),
-        JobKind::Script => script_path_to_payload(job.script_path(), &mut tx, &w_id).await?,
+        JobKind::Script => {
+            script_path_to_payload(
+                job.script_path(),
+                &mut tx,
+                &w_id,
+                run_query.skip_preprocessor,
+            )
+            .await?
+        }
         _ => return Err(anyhow::anyhow!("Not supported").into()),
     };
 
@@ -3145,6 +3178,17 @@ impl Drop for Guard {
     }
 }
 
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+    pub static ref TIMEOUT_WAIT_RESULT: Arc<RwLock<Option<u64>>> = Arc::new(RwLock::new(
+        std::env::var("TIMEOUT_WAIT_RESULT")
+            .ok()
+            .and_then(|x| x.parse::<u64>().ok())
+    ));
+}
+
 #[derive(Deserialize)]
 pub struct WindmillCompositeResult {
     windmill_status_code: Option<u16>,
@@ -3159,7 +3203,7 @@ async fn run_wait_result(
     username: &str,
 ) -> error::Result<Response> {
     let mut result = None;
-    let timeout = SERVER_CONFIG.read().await.timeout_wait_result.clone();
+    let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600);
     let timeout_ms = if timeout <= 0 {
         2000
     } else {
@@ -3451,7 +3495,7 @@ pub async fn run_wait_result_job_by_path_get(
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
     let (job_payload, tag, delete_after_use, timeout) =
-        script_path_to_payload(script_path, &mut tx, &w_id).await?;
+        script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
 
     let tag = run_query.tag.clone().or(tag);
     check_tag_available_for_workspace(&w_id, &tag).await?;
@@ -3527,7 +3571,7 @@ pub async fn run_wait_result_flow_by_path_get(
     let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() };
 
     run_wait_result_flow_by_path_internal(
-        db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
+        db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None,
     )
     .await
 }
@@ -3553,11 +3597,12 @@ pub async fn run_wait_result_script_by_path(
         user_db,
         w_id,
         args,
+        None,
     )
     .await
 }
 
-async fn run_wait_result_script_by_path_internal(
+pub async fn run_wait_result_script_by_path_internal(
     db: sqlx::Pool<Postgres>,
     run_query: RunJobQuery,
     script_path: StripPath,
@@ -3566,6 +3611,7 @@ async fn run_wait_result_script_by_path_internal(
     user_db: UserDB,
     w_id: String,
     args: PushArgsOwned,
+    label_prefix: Option<String>,
 ) -> error::Result<Response> {
     check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
@@ -3574,7 +3620,7 @@ async fn run_wait_result_script_by_path_internal(
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
 
     let (job_payload, tag, delete_after_use, timeout) =
-        script_path_to_payload(script_path, &mut tx, &w_id).await?;
+        script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
 
     let tag = run_query.tag.clone().or(tag);
     check_tag_available_for_workspace(&w_id, &tag).await?;
@@ -3587,7 +3633,9 @@ async fn run_wait_result_script_by_path_internal(
         &w_id,
         job_payload,
         PushArgs { args: &args.args, extra: args.extra },
-        authed.display_username(),
+        &label_prefix
+            .map(|x| x + authed.display_username())
+            .unwrap_or_else(|| authed.display_username().to_string()),
         &authed.email,
         username_to_permissioned_as(&authed.username),
         None,
@@ -3644,6 +3692,7 @@ pub async fn run_wait_result_script_by_hash(
         priority,
         delete_after_use,
         timeout,
+        has_preprocessor,
     ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?;
     if let Some(run_query_cache_ttl) = run_query.cache_ttl {
         cache_ttl = Some(run_query_cache_ttl);
@@ -3669,6 +3718,8 @@ pub async fn run_wait_result_script_by_hash(
             language,
             dedicated_worker,
             priority,
+            apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
+                && has_preprocessor.unwrap_or(false),
         },
         PushArgs { args: &args.args, extra: args.extra },
         authed.display_username(),
@@ -3712,12 +3763,12 @@ pub async fn run_wait_result_flow_by_path(
     check_license_key_valid().await?;
 
     run_wait_result_flow_by_path_internal(
-        db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
+        db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None,
     )
     .await
 }
 
-async fn run_wait_result_flow_by_path_internal(
+pub async fn run_wait_result_flow_by_path_internal(
     db: sqlx::Pool<Postgres>,
     run_query: RunJobQuery,
     flow_path: StripPath,
@@ -3726,6 +3777,7 @@ async fn run_wait_result_flow_by_path_internal(
     user_db: UserDB,
     args: PushArgsOwned,
     w_id: String,
+    label_prefix: Option<String>,
 ) -> error::Result<Response> {
     check_queue_too_long(&db, run_query.queue_limit).await?;
 
@@ -3736,8 +3788,8 @@ async fn run_wait_result_flow_by_path_internal(
 
     let scheduled_for = run_query.get_scheduled_for(&db).await?;
 
-    let (tag, dedicated_worker, early_return) = sqlx::query!(
-        "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return 
+    let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!(
+        "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor
         FROM flow 
         LEFT JOIN flow_version
             ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
@@ -3747,7 +3799,7 @@ async fn run_wait_result_flow_by_path_internal(
     )
     .fetch_optional(&mut tx)
     .await?
-    .map(|x| (x.tag, x.dedicated_worker, x.early_return))
+    .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor))
     .ok_or_else(|| {
         Error::NotFound(format!(
             "flow not found at path {flow_path} in workspace {w_id}"
@@ -3763,9 +3815,16 @@ async fn run_wait_result_flow_by_path_internal(
         &db,
         tx,
         &w_id,
-        JobPayload::Flow { path: flow_path.to_string(), dedicated_worker },
+        JobPayload::Flow {
+            path: flow_path.to_string(),
+            dedicated_worker,
+            apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
+                && has_preprocessor.unwrap_or(false),
+        },
         PushArgs { args: &args.args, extra: args.extra },
-        authed.display_username(),
+        &label_prefix
+            .map(|x| x + authed.display_username())
+            .unwrap_or_else(|| authed.display_username().to_string()),
         &authed.email,
         username_to_permissioned_as(&authed.username),
         scheduled_for,
@@ -3801,7 +3860,7 @@ async fn run_preview_script(
     #[cfg(feature = "enterprise")]
     check_license_key_valid().await?;
 
-    check_scopes(&authed, || format!("runscript"))?;
+    check_scopes(&authed, || format!("jobs:runscript"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -3871,7 +3930,7 @@ async fn run_bundle_preview_script(
 
     check_license_key_valid().await?;
 
-    check_scopes(&authed, || format!("runscript"))?;
+    check_scopes(&authed, || format!("jobs:runscript"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -4202,6 +4261,7 @@ async fn add_batch_jobs(
                     _priority,
                     _delete_after_use,
                     timeout,
+                    _,
                 ) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?;
                 (
                     Some(script_hash),
@@ -4228,7 +4288,11 @@ async fn add_batch_jobs(
                 JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None }
             } else {
                 if let Some(path) = batch_info.path.as_ref() {
-                    JobPayload::Flow { path: path.to_string(), dedicated_worker: None }
+                    JobPayload::Flow {
+                        path: path.to_string(),
+                        dedicated_worker: None,
+                        apply_preprocessor: false,
+                    }
                 } else {
                     Err(anyhow::anyhow!(
                         "Path is required if no value is not provided"
@@ -4352,7 +4416,7 @@ async fn run_preview_flow_job(
     Query(run_query): Query<RunJobQuery>,
     Json(raw_flow): Json<PreviewFlow>,
 ) -> error::Result<(StatusCode, String)> {
-    check_scopes(&authed, || format!("runflow"))?;
+    check_scopes(&authed, || format!("jobs:runflow"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -4450,6 +4514,7 @@ pub async fn run_job_by_hash_inner(
         priority,
         _delete_after_use, // not taken into account in async endpoints
         timeout,
+        has_preprocessor,
     ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?;
     check_scopes(&authed, || format!("run:script/{path}"))?;
     if let Some(run_query_cache_ttl) = run_query.cache_ttl {
@@ -4475,6 +4540,8 @@ pub async fn run_job_by_hash_inner(
             language,
             dedicated_worker,
             priority,
+            apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
+                && has_preprocessor.unwrap_or(false),
         },
         PushArgs { args: &args.args, extra: args.extra },
         &label_prefix
@@ -4609,8 +4676,8 @@ async fn get_job_update(
                 "progress_perc"
                 
             )
-            .fetch_one(&db)
-            .await?
+            .fetch_optional(&db)
+            .await?.and_then(|inner| inner)
     } else {
         None
     };
@@ -4789,14 +4856,19 @@ pub fn list_completed_jobs_query(
     lq: &ListCompletedQuery,
     fields: &[&str],
     join_outstanding_wait_times: bool,
+    tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
-    let sqlb = SqlBuilder::select_from("completed_job")
+    let mut sqlb = SqlBuilder::select_from("completed_job")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .offset(offset)
         .limit(per_page)
         .clone();
 
+    if let Some(tags) = tags {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times)
 }
 #[derive(Deserialize, Clone)]
@@ -4841,7 +4913,7 @@ async fn list_completed_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<ListableCompletedJob>> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     let (per_page, offset) = paginate(pagination);
 
@@ -4883,6 +4955,7 @@ async fn list_completed_jobs(
             "'CompletedJob' as type",
         ],
         false,
+        get_scope_tags(&authed),
     )
     .sql()?;
     let mut tx = user_db.begin(&authed).await?;
@@ -5107,7 +5180,7 @@ async fn delete_completed_job<'a>(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, id)): Path<(String, Uuid)>,
 ) -> error::Result<Response> {
-    check_scopes(&authed, || format!("deletejob"))?;
+    check_scopes(&authed, || format!("jobs:deletejob"))?;
 
     let mut tx = user_db.begin(&authed).await?;
 

backend/windmill-api/src/lib.rs

@@ -60,6 +60,7 @@ mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
+mod http_triggers;
 mod indexer_ee;
 mod inputs;
 mod integration;
@@ -283,7 +284,8 @@ pub async fn run_server(
                         )
                         .nest("/variables", variables::workspaced_service())
                         .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/oidc", oidc_ee::workspaced_service()),
+                        .nest("/oidc", oidc_ee::workspaced_service())
+                        .nest("/http_triggers", http_triggers::workspaced_service()),
                 )
                 .nest("/workspaces", workspaces::global_service())
                 .nest(
@@ -336,7 +338,7 @@ pub async fn run_server(
                 )
                 .nest(
                     "/w/:workspace_id/capture_u",
-                    capture::global_service().layer(cors),
+                    capture::global_service().layer(cors.clone()),
                 )
                 .nest(
                     "/auth",
@@ -346,6 +348,10 @@ pub async fn run_server(
                     "/oauth",
                     oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))),
                 )
+                .nest(
+                    "/r",
+                    http_triggers::routes_global_service().layer(from_extractor::<OptAuthed>()),
+                )
                 .route("/version", get(git_v))
                 .route("/uptodate", get(is_up_to_date))
                 .route("/ee_license", get(ee_license))

backend/windmill-api/src/scripts.rs

@@ -103,6 +103,8 @@ pub struct ScriptWDraft {
     pub visible_to_runner_only: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub no_main_func: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_preprocessor: Option<bool>,
 }
 
 pub fn global_service() -> Router {
@@ -581,6 +583,7 @@ async fn create_script_internal<'c>(
         || ns.language == ScriptLang::Bunnative
         || ns.language == ScriptLang::Deno
         || ns.language == ScriptLang::Rust
+        || ns.language == ScriptLang::Ansible
         || ns.language == ScriptLang::Php)
     {
         Some(String::new())
@@ -612,8 +615,8 @@ async fn create_script_internal<'c>(
          content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
          draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
          dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
-         delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)",
+         delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)",
         &w_id,
         &hash.0,
         ns.path,
@@ -643,7 +646,8 @@ async fn create_script_internal<'c>(
         ns.concurrency_key,
         ns.visible_to_runner_only,
         ns.no_main_func,
-        codebase
+        codebase,
+        ns.has_preprocessor,
     )
     .execute(&mut tx)
     .await?;
@@ -738,6 +742,7 @@ async fn create_script_internal<'c>(
             },
         );
     }
+
     let permissioned_as = username_to_permissioned_as(&authed.username);
     if needs_lock_gen {
         let tag = if ns.dedicated_worker.is_some_and(|x| x) {
@@ -878,7 +883,7 @@ async fn get_script_by_path_w_draft(
     let mut tx = user_db.begin(&authed).await?;
 
     let script_o = sqlx::query_as::<_, ScriptWDraft>(
-        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func FROM script LEFT JOIN draft ON 
+        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor FROM script LEFT JOIN draft ON 
          script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
          WHERE script.path = $1 AND script.workspace_id = $2 \
          AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \

backend/windmill-api/src/settings.rs

@@ -24,6 +24,8 @@ use axum::{
 #[cfg(feature = "enterprise")]
 use axum::extract::Query;
 
+#[cfg(feature = "enterprise")]
+use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
 use serde::Deserialize;
 use windmill_common::{
     error::{self, JsonResult, Result},
@@ -55,7 +57,8 @@ pub fn global_service() -> Router {
             get(get_latest_key_renewal_attempt),
         )
         .route("/renew_license_key", post(renew_license_key))
-        .route("/customer_portal", post(create_customer_portal_session));
+        .route("/customer_portal", post(create_customer_portal_session))
+        .route("/test_critical_channels", post(test_critical_channels));
 
     #[cfg(feature = "parquet")]
     {
@@ -387,3 +390,27 @@ pub async fn create_customer_portal_session(
 
     return Ok(url);
 }
+
+#[cfg(feature = "enterprise")]
+pub async fn test_critical_channels(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+    Json(test_critical_channels): Json<Vec<CriticalErrorChannel>>,
+) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+
+    #[cfg(feature = "enterprise")]
+    send_critical_alert(
+        "Test critical error".to_string(),
+        &db,
+        CriticalAlertKind::CriticalError,
+        Some(test_critical_channels),
+    )
+    .await;
+    Ok("Sent test critical error".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn test_critical_channels() -> Result<String> {
+    Ok("Critical channels require EE".to_string())
+}

backend/windmill-api/src/users.rs

@@ -51,7 +51,7 @@ use windmill_audit::ActionKind;
 use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING;
 use windmill_common::users::truncate_token;
 use windmill_common::utils::{paginate, send_email};
-use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG};
+use windmill_common::worker::{CLOUD_HOSTED, SMTP_CONFIG};
 use windmill_common::{
     auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET},
     db::UserDB,
@@ -651,9 +651,12 @@ where
                 {
                     if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
                         parts.extensions.insert(authed.clone());
-                        if authed.scopes.is_some()
-                            && (path_vec.len() < 3
-                                || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
+                        if authed.scopes.as_ref().is_some_and(|scopes| {
+                            scopes
+                                .iter()
+                                .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
+                        }) && (path_vec.len() < 3
+                            || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
                         {
                             BRUTE_FORCE_COUNTER.increment().await;
                             return Err((
@@ -681,15 +684,30 @@ pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
 where
     F: FnOnce() -> String,
 {
-    if let Some(scopes) = &authed.scopes {
+    if authed.scopes.as_ref().is_some_and(|scopes| {
+        scopes
+            .iter()
+            .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
+    }) {
         let req = &required();
-        if !scopes.contains(req) {
+        if !authed.scopes.as_ref().unwrap().contains(req) {
             return Err(Error::BadRequest(format!("missing required scope: {req}")));
         }
     }
     Ok(())
 }
 
+pub fn get_scope_tags(authed: &ApiAuthed) -> Option<Vec<&str>> {
+    authed
+        .scopes
+        .as_ref()?
+        .iter()
+        .find_map(|s| match s.split(":").collect::<Vec<_>>().as_slice() {
+            ["if_jobs", "filter_tags", tags] => Some(tags.split(",").collect::<Vec<_>>()),
+            _ => None,
+        })
+}
+
 #[derive(Clone, Debug)]
 pub struct OptAuthed(pub Option<ApiAuthed>);
 
@@ -1966,7 +1984,7 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
 }
 
 pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: String) -> Result<()> {
-    if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() {
+    if let Some(smtp) = SMTP_CONFIG.read().await.clone() {
         send_email(subject, content, vec![to], smtp, None).await?;
     }
     return Ok(());

backend/windmill-api/src/workspaces.rs

@@ -2317,6 +2317,8 @@ struct ScriptMetadata {
     pub codebase: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub concurrency_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_preprocessor: Option<bool>,
 }
 
 pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -2575,7 +2577,7 @@ async fn tarball_workspace(
                 }
                 ScriptLang::Php => "php",
                 ScriptLang::Rust => "rs",
-                ScriptLang::Ansible => "yaml",
+                ScriptLang::Ansible => "playbook.yml",
             };
             archive
                 .write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
@@ -2602,6 +2604,7 @@ async fn tarball_workspace(
                 no_main_func: script.no_main_func,
                 codebase: script.codebase,
                 concurrency_key: script.concurrency_key,
+                has_preprocessor: script.has_preprocessor,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
             archive

backend/windmill-common/src/auth.rs

@@ -25,6 +25,7 @@ pub struct JWTAuthClaims {
     pub workspace_id: String,
     pub exp: usize,
     pub job_id: Option<String>,
+    pub scopes: Option<Vec<String>>,
 }
 
 #[derive(Deserialize)]

backend/windmill-common/src/ee.rs

@@ -26,7 +26,10 @@ pub async fn get_license_plan() -> LicensePlan {
 
 #[derive(Deserialize)]
 #[serde(untagged)]
-pub enum CriticalErrorChannel {}
+pub enum CriticalErrorChannel {
+    Email { email: String },
+    Slack { slack_channel: String },
+}
 
 pub enum CriticalAlertKind {
     #[cfg(feature = "enterprise")]
@@ -36,7 +39,13 @@ pub enum CriticalAlertKind {
 }
 
 #[cfg(feature = "enterprise")]
-pub async fn send_critical_alert(_error_message: String, _db: &DB, _kind: CriticalAlertKind) {}
+pub async fn send_critical_alert(
+    _error_message: String,
+    _db: &DB,
+    _kind: CriticalAlertKind,
+    _channels: Option<Vec<CriticalErrorChannel>>,
+) {
+}
 
 #[cfg(feature = "enterprise")]
 pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {

backend/windmill-common/src/flow_status.rs

@@ -29,6 +29,7 @@ pub struct FlowStatus {
     pub step: i32,
     pub modules: Vec<FlowStatusModule>,
     pub failure_module: Box<FlowStatusModuleWParent>,
+    pub preprocessor_module: Option<FlowStatusModule>,
 
     #[serde(skip_serializing_if = "HashMap::is_empty")]
     #[serde(default)]
@@ -355,7 +356,11 @@ impl FlowStatusModule {
 impl FlowStatus {
     pub fn new(f: &FlowValue) -> Self {
         Self {
-            step: 0,
+            step: if f.preprocessor_module.is_some() {
+                -1
+            } else {
+                0
+            },
             approval_conditions: None,
             modules: f
                 .modules
@@ -372,6 +377,13 @@ impl FlowStatus {
                         .unwrap_or_else(|| "failure".to_string()),
                 },
             }),
+            preprocessor_module: if f.preprocessor_module.is_some() {
+                Some(FlowStatusModule::WaitingForPriorSteps {
+                    id: f.preprocessor_module.as_ref().unwrap().id.clone(),
+                })
+            } else {
+                None
+            },
             cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] },
             retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
             restarted_from: None,

backend/windmill-common/src/flows.rs

@@ -102,6 +102,8 @@ pub struct FlowValue {
     #[serde(default)]
     pub failure_module: Option<Box<FlowModule>>,
     #[serde(default)]
+    pub preprocessor_module: Option<Box<FlowModule>>,
+    #[serde(default)]
     #[serde(skip_serializing_if = "is_default")]
     pub same_worker: bool,
 

backend/windmill-common/src/global_settings.rs

@@ -14,6 +14,8 @@ pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
 pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
 pub const SCIM_TOKEN_SETTING: &str = "scim_token";
 pub const SAML_METADATA_SETTING: &str = "saml_metadata";
+pub const SMTP_SETTING: &str = "smtp_settings";
+pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
 
 pub const UNIQUE_ID_SETTING: &str = "uid";
 pub const DISABLE_STATS_SETTING: &str = "disable_stats";

backend/windmill-common/src/jobs.rs

@@ -11,6 +11,8 @@ use uuid::Uuid;
 
 pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
 
+pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR";
+
 use crate::{
     error::{self, to_anyhow, Error},
     flow_status::{FlowStatus, RestartedFrom},
@@ -302,6 +304,7 @@ pub enum JobPayload {
         dedicated_worker: Option<bool>,
         language: ScriptLang,
         priority: Option<i16>,
+        apply_preprocessor: bool,
     },
     Code(RawCode),
     Dependencies {
@@ -331,6 +334,7 @@ pub enum JobPayload {
     Flow {
         path: String,
         dedicated_worker: Option<bool>,
+        apply_preprocessor: bool,
     },
     RestartedFlow {
         completed_job_id: Uuid,
@@ -383,6 +387,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
     script_path: &str,
     db: E,
     w_id: &str,
+    skip_preprocessor: Option<bool>,
 ) -> error::Result<(JobPayload, Option<Tag>, Option<bool>, Option<i32>)> {
     let (job_payload, tag, delete_after_use, script_timeout) = if script_path.starts_with("hub/") {
         (
@@ -404,6 +409,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
             priority,
             delete_after_use,
             script_timeout,
+            has_preprocessor,
         ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
         (
             JobPayload::ScriptHash {
@@ -416,6 +422,8 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
                 language,
                 dedicated_worker,
                 priority,
+                apply_preprocessor: !skip_preprocessor.unwrap_or(false)
+                    && has_preprocessor.unwrap_or(false),
             },
             tag,
             delete_after_use,
@@ -473,7 +481,7 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa
     w_id: &str,
 ) -> Result<(JobPayload, Option<String>), Error> {
     let (payload, tag, _, _) = if path.starts_with("script/") {
-        script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await?
+        script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await?
     } else if path.starts_with("flow/") {
         let path = path.strip_prefix("flow/").unwrap().to_string();
         let r = sqlx::query!(
@@ -486,7 +494,12 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa
         let (tag, dedicated_worker) = r
             .map(|x| (x.tag, x.dedicated_worker))
             .unwrap_or_else(|| (None, None));
-        (JobPayload::Flow { path, dedicated_worker }, tag, None, None)
+        (
+            JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false },
+            tag,
+            None,
+            None,
+        )
     } else {
         return Err(Error::BadRequest(format!(
             "path must start with script/ or flow/ (got {})",

backend/windmill-common/src/lib.rs

@@ -271,9 +271,10 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas
     Option<i16>,
     Option<bool>,
     Option<i32>,
+    Option<bool>,
 )> {
     let r_o = sqlx::query!(
-        "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND
+        "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND
     created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
     deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
         script_path,
@@ -296,6 +297,7 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas
         script.priority,
         script.delete_after_use,
         script.timeout,
+        script.has_preprocessor,
     ))
 }
 

backend/windmill-common/src/scripts.rs

@@ -193,6 +193,8 @@ pub struct Script {
     pub no_main_func: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub codebase: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_preprocessor: Option<bool>,
 }
 
 #[derive(Serialize, sqlx::FromRow)]
@@ -289,6 +291,7 @@ pub struct NewScript {
     pub visible_to_runner_only: Option<bool>,
     pub no_main_func: Option<bool>,
     pub codebase: Option<String>,
+    pub has_preprocessor: Option<bool>,
 }
 
 fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>

backend/windmill-common/src/server.rs

@@ -13,22 +13,20 @@ pub struct Smtp {
 }
 
 #[derive(Serialize, Deserialize, PartialEq)]
-pub struct ServerConfigOpt {
+pub struct SmtpConfigOpt {
     pub smtp_host: Option<String>,
     pub smtp_username: Option<String>,
     pub smtp_password: Option<String>,
     pub smtp_port: Option<u16>,
     pub smtp_from: Option<String>,
     pub smtp_tls_implicit: Option<bool>,
-    pub timeout_wait_result: Option<u64>,
 }
 
-pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
-    let config: ServerConfigOpt =
-        sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",)
+pub async fn load_smtp_config(db: &DB) -> error::Result<Option<Smtp>> {
+    let config: SmtpConfigOpt =
+        sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'smtp_settings'",)
             .fetch_optional(db)
             .await?
-            .flatten()
             .map(|x| serde_json::from_value(x).ok())
             .flatten()
             .unwrap_or_default();
@@ -77,20 +75,10 @@ pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
         tracing::warn!("SMTP not configured");
     }
 
-    Ok(ServerConfig {
-        smtp,
-        timeout_wait_result: config
-            .timeout_wait_result
-            .ok_or(
-                std::env::var("TIMEOUT_WAIT_RESULT")
-                    .ok()
-                    .and_then(|x| x.parse::<u64>().ok()),
-            )
-            .unwrap_or(600),
-    })
+    Ok(smtp)
 }
 
-impl Default for ServerConfigOpt {
+impl Default for SmtpConfigOpt {
     fn default() -> Self {
         Self {
             smtp_from: None,
@@ -99,13 +87,6 @@ impl Default for ServerConfigOpt {
             smtp_port: None,
             smtp_tls_implicit: None,
             smtp_username: None,
-            timeout_wait_result: Default::default(),
         }
     }
 }
-
-#[derive(PartialEq, Clone, Debug)]
-pub struct ServerConfig {
-    pub smtp: Option<Smtp>,
-    pub timeout_wait_result: u64,
-}

backend/windmill-common/src/utils.rs

@@ -288,7 +288,7 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
     }
 
     #[cfg(feature = "enterprise")]
-    send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError).await;
+    send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
 }
 
 pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
@@ -304,5 +304,11 @@ pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
         tracing::error!("Failed to save critical error to database: {}", err);
     }
     #[cfg(feature = "enterprise")]
-    send_critical_alert(message, &_db, CriticalAlertKind::RecoveredCriticalError).await;
+    send_critical_alert(
+        message,
+        &_db,
+        CriticalAlertKind::RecoveredCriticalError,
+        None,
+    )
+    .await;
 }

backend/windmill-common/src/worker.rs

@@ -14,7 +14,7 @@ use std::{
 };
 use tokio::sync::RwLock;
 
-use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
+use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB};
 
 lazy_static::lazy_static! {
     pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -66,8 +66,7 @@ lazy_static::lazy_static! {
     pub static ref WORKER_SUSPENDED_PULL_QUERY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 
 
-    pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
-
+    pub static ref SMTP_CONFIG: Arc<RwLock<Option<Smtp>>> = Arc::new(RwLock::new(None));
 
 
     pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
@@ -195,7 +194,11 @@ fn normalize_path(path: &Path) -> PathBuf {
     }
     ret
 }
-pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &str, content: &str) -> error::Result<File> {
+pub fn write_file_at_user_defined_location(
+    job_dir: &str,
+    user_defined_path: &str,
+    content: &str,
+) -> error::Result<PathBuf> {
     let job_dir = Path::new(job_dir);
     let user_path = PathBuf::from(user_defined_path);
 
@@ -210,9 +213,11 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st
         return Err(std::io::Error::new(
             std::io::ErrorKind::PermissionDenied,
             "Path is outside the allowed job directory.",
-        ).into());
+        )
+        .into());
     }
 
+    let full_path = normalized_full_path.as_path();
     if let Some(parent_dir) = full_path.parent() {
         std::fs::create_dir_all(parent_dir)?;
     }
@@ -220,7 +225,7 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st
     let mut file = File::create(full_path)?;
     file.write_all(content.as_bytes())?;
     file.flush()?;
-    Ok(file)
+    Ok(normalized_full_path)
 }
 
 pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {

backend/windmill-queue/src/jobs.rs

@@ -52,6 +52,7 @@ use windmill_common::{
     },
     jobs::{
         get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode,
+        ENTRYPOINT_OVERRIDE, PREPROCESSOR_FAKE_ENTRYPOINT,
     },
     schedule::Schedule,
     scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
@@ -474,6 +475,11 @@ where
     }
 }
 
+#[derive(Deserialize)]
+struct RawFlowFailureModule {
+    failure_module: Option<Box<RawValue>>,
+}
+
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
     db: &Pool<Postgres>,
@@ -921,7 +927,17 @@ pub async fn add_completed_job<
             )
             .await;
         } else if !skip_downstream_error_handlers
-            && matches!(queued_job.job_kind, JobKind::Flow | JobKind::Script)
+            && (matches!(queued_job.job_kind, JobKind::Script)
+                || matches!(queued_job.job_kind, JobKind::Flow)
+                    && queued_job
+                        .raw_flow
+                        .as_ref()
+                        .and_then(|v| {
+                            serde_json::from_str::<RawFlowFailureModule>((**v).get())
+                                .ok()
+                                .and_then(|v| v.failure_module)
+                        })
+                        .is_none())
             && queued_job.parent_job.is_none()
         {
             let result = serde_json::from_str(
@@ -1021,6 +1037,7 @@ pub async fn add_completed_job<
                             .clone()
                             .unwrap_or_else(|| ScriptLang::Deno),
                         priority: queued_job.priority,
+                        apply_preprocessor: false,
                     },
                     queued_job
                         .args
@@ -2864,10 +2881,21 @@ where
             (content_type, extra, raw, wrap_body)
         };
 
-        if content_type.is_none() || content_type.unwrap().starts_with("application/json") {
+        let no_content_type = content_type.is_none();
+        if no_content_type || content_type.unwrap().starts_with("application/json") {
             let bytes = Bytes::from_request(req, _state)
                 .await
                 .map_err(IntoResponse::into_response)?;
+            if no_content_type && bytes.is_empty() {
+                if use_raw {
+                    extra.insert("raw_string".to_string(), to_raw_value(&"".to_string()));
+                }
+                let mut args = HashMap::new();
+                if wrap_body {
+                    args.insert("body".to_string(), to_raw_value(&serde_json::json!({})));
+                }
+                return Ok(PushArgsOwned { extra: Some(extra), args: args });
+            }
             let str = String::from_utf8(bytes.to_vec())
                 .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
 
@@ -2992,7 +3020,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
     mut tx: PushIsolationLevel<'c, R>,
     workspace_id: &str,
     job_payload: JobPayload,
-    args: PushArgs<'d>,
+    mut args: PushArgs<'d>,
     user: &str,
     mut email: &str,
     mut permissioned_as: String,
@@ -3211,21 +3239,38 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
             language,
             dedicated_worker,
             priority,
-        } => (
-            Some(hash.0),
-            Some(path),
-            None,
-            JobKind::Script,
-            None,
-            None,
-            Some(language),
-            custom_concurrency_key,
-            concurrent_limit,
-            concurrency_time_window_s,
-            cache_ttl,
-            dedicated_worker,
-            priority,
-        ),
+            apply_preprocessor,
+        } => {
+            let extra = args.extra.get_or_insert_with(HashMap::new);
+            if apply_preprocessor {
+                extra.insert(
+                    ENTRYPOINT_OVERRIDE.to_string(),
+                    to_raw_value(&PREPROCESSOR_FAKE_ENTRYPOINT),
+                );
+                extra.entry("wm_trigger".to_string()).or_insert_with(|| {
+                    to_raw_value(&serde_json::json!({
+                        "kind": "webhook",
+                    }))
+                });
+            } else {
+                extra.remove("wm_trigger");
+            }
+            (
+                Some(hash.0),
+                Some(path),
+                None,
+                JobKind::Script,
+                None,
+                None,
+                Some(language),
+                custom_concurrency_key,
+                concurrent_limit,
+                concurrency_time_window_s,
+                cache_ttl,
+                dedicated_worker,
+                priority,
+            )
+        }
         JobPayload::ScriptHub { path } => {
             if path == "hub/7771/slack" || path == "hub/7836/slack" {
                 permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
@@ -3404,9 +3449,13 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                             branch_or_iteration_n: restarted_from_val.branch_or_iteration_n,
                         }),
                         user_states,
+                        preprocessor_module: None,
                     }
                 }
-                _ => FlowStatus::new(&value), // this is a new flow being pushed, flow_status is set to flow_value
+                _ => {
+                    value.preprocessor_module = None;
+                    FlowStatus::new(&value)
+                } // this is a new flow being pushed, flow_status is set to flow_value
             };
             (
                 None,
@@ -3473,6 +3522,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                 early_return: None,
                 concurrency_key: custom_concurrency_key.clone(),
                 priority: priority,
+                preprocessor_module: None,
             };
             (
                 None,
@@ -3490,7 +3540,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                 priority,
             )
         }
-        JobPayload::Flow { path, dedicated_worker } => {
+        JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => {
             let value_json = fetch_scalar_isolated!(
                 sqlx::query_as::<_, FlowRawValue>(
                     "SELECT flow_version.value FROM flow 
@@ -3518,6 +3568,18 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
             let custom_concurrency_key = value.concurrency_key.clone();
             let concurrency_time_window_s = value.concurrency_time_window_s.clone();
             let concurrent_limit = value.concurrent_limit.clone();
+
+            let extra = args.extra.get_or_insert_with(HashMap::new);
+            if !apply_preprocessor {
+                value.preprocessor_module = None;
+                extra.remove("wm_trigger");
+            } else {
+                extra.entry("wm_trigger".to_string()).or_insert_with(|| {
+                    to_raw_value(&serde_json::json!({
+                        "kind": "webhook",
+                    }))
+                });
+            }
             let status = Some(FlowStatus::new(&value));
             (
                 None,
@@ -3574,6 +3636,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
                     branch_or_iteration_n,
                 }),
                 user_states,
+                preprocessor_module: None,
             };
             (
                 None,

backend/windmill-queue/src/schedule.rs

@@ -120,7 +120,11 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             .map(|x| (x.tag, x.dedicated_worker))
             .unwrap_or_else(|| (None, None));
         (
-            JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker },
+            JobPayload::Flow {
+                path: schedule.script_path.clone(),
+                dedicated_worker,
+                apply_preprocessor: false,
+            },
             tag,
             None,
         )
@@ -188,6 +192,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                     dedicated_worker,
                     language,
                     priority,
+                    apply_preprocessor: false,
                 },
                 if schedule.tag.as_ref().is_some_and(|x| x != "") {
                     schedule.tag.clone()

backend/windmill-worker/nsjail/run.ansible.config.proto

@@ -0,0 +1,140 @@
+name: "ansible run script"
+
+mode: ONCE
+hostname: "ansible"
+log_level: ERROR
+
+rlimit_as: 4096
+rlimit_cpu: 1000
+rlimit_fsize: 1000
+rlimit_nofile: 10000
+
+cwd: "/tmp"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+keep_caps: false
+keep_env: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+    is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+    is_bind: true
+}
+
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+    is_bind: true
+}
+
+mount {
+    src: "/dev/null"
+    dst: "/dev/null"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    dst: "/dev/shm"
+    fstype: "tmpfs"
+    rw: true
+    is_bind: false
+}
+
+mount {
+    dst: "/tmp"
+    fstype: "tmpfs"
+    rw: true
+    options: "size=800000000"
+}
+
+mount {
+    src: "{JOB_DIR}/main.yml"
+    dst: "/tmp/main.yml"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/wrapper.sh"
+    dst: "/tmp/wrapper.sh"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/requirements.yml"
+    dst: "/tmp/requirements.yml"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "{JOB_DIR}/ansible.cfg"
+    dst: "/tmp/ansible.cfg"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/ansible_collections/"
+    dst: "/tmp/ansible_collections/"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/args.json"
+    dst: "/tmp/args.json"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/result.json"
+    dst: "/tmp/result_nsjail_mount.json"
+    rw: true
+    is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+mount {
+    src: "/dev/random"
+    dst: "/dev/random"
+    is_bind: true
+}
+
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
+{SHARED_MOUNT}
+
+{SHARED_DEPENDENCIES}
+
+{FILE_RESOURCES}
+
+iface_no_lo: true
+
+envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
+envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}"
+envar: "HOME=/tmp"
+envar: "ANSIBLE_CONFIG=/tmp/ansible.cfg"

backend/windmill-worker/src/ansible_executor.rs

@@ -1,24 +1,32 @@
-use std::{collections::HashMap, process::Stdio};
+use std::{
+    collections::HashMap,
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+    process::Stdio,
+};
 
 use anyhow::anyhow;
+use itertools::Itertools;
 use serde_json::value::RawValue;
 use tokio::process::Command;
 use uuid::Uuid;
 use windmill_common::{
     error,
     jobs::QueuedJob,
-    worker::{write_file, write_file_at_user_defined_location, WORKER_CONFIG},
+    worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
 };
 use windmill_parser_yaml::AnsibleRequirements;
 use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
+    bash_executor::BIN_BASH,
     common::{
-        get_reserved_variables, handle_child, read_and_check_result,
-        start_child_process, transform_json,
+        get_reserved_variables, handle_child, read_and_check_result, start_child_process,
+        transform_json,
     },
     python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
-    AuthedClientBackgroundTask, DISABLE_NSJAIL, HOME_ENV, PATH_ENV, TZ_ENV,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    TZ_ENV,
 };
 
 lazy_static::lazy_static! {
@@ -29,6 +37,8 @@ lazy_static::lazy_static! {
     std::env::var("ANSIBLE_GALAXY_PATH").unwrap_or("/usr/local/bin/ansible-galaxy".to_string());
 }
 
+const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto");
+
 async fn handle_ansible_python_deps(
     job_dir: &str,
     requirements_o: Option<String>,
@@ -98,6 +108,7 @@ async fn handle_ansible_python_deps(
     }
     Ok(additional_python_paths)
 }
+
 async fn install_galaxy_collections(
     collections_yml: &str,
     job_dir: &str,
@@ -155,6 +166,24 @@ async fn install_galaxy_collections(
     Ok(())
 }
 
+#[cfg(not(feature = "enterprise"))]
+fn check_ansible_exists() -> Result<(), error::Error> {
+    if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
+        let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
+        return Err(error::Error::NotFound(msg));
+    }
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+fn check_ansible_exists() -> Result<(), error::Error> {
+    if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
+        let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
+        return Err(error::Error::NotFound(msg));
+    }
+    Ok(())
+}
+
 pub async fn handle_ansible_job(
     requirements_o: Option<String>,
     job_dir: &str,
@@ -166,20 +195,16 @@ pub async fn handle_ansible_job(
     db: &sqlx::Pool<sqlx::Postgres>,
     client: &AuthedClientBackgroundTask,
     inner_content: &String,
+    shared_mount: &str,
     base_internal_url: &str,
     envs: HashMap<String, String>,
 ) -> windmill_common::error::Result<Box<RawValue>> {
+    check_ansible_exists()?;
+
     let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?;
     append_logs(&job.id, &job.workspace_id, logs, db).await;
     write_file(job_dir, "main.yml", &playbook)?;
 
-    let ansible_cfg_content = r#"
-[defaults]
-collections_path = ./
-roles_path = ./roles
-"#;
-    write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
-
     let additional_python_paths = handle_ansible_python_deps(
         job_dir,
         requirements_o,
@@ -196,7 +221,13 @@ roles_path = ./roles
 
     let interpolated_args;
     if let Some(args) = &job.args {
-        if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
+        let mut args = args.0.clone();
+        if let Some(reqs) = reqs.clone() {
+            for (name, path) in reqs.resources.iter().chain(reqs.vars.iter()) {
+                args.insert(name.clone(), to_raw_value(path));
+            }
+        }
+        if let Some(x) = transform_json(client, &job.workspace_id, &args, job, db).await? {
             write_file(
                 job_dir,
                 "args.json",
@@ -209,12 +240,19 @@ roles_path = ./roles
                 "args.json",
                 &serde_json::to_string(&args).unwrap_or_else(|_| "{}".to_string()),
             )?;
-            interpolated_args = Some(args.clone().0);
+            interpolated_args = Some(args);
         }
     } else {
         interpolated_args = None;
         write_file(job_dir, "args.json", "{}")?;
     };
+    write_file(job_dir, "result.json", "")?;
+
+    let cmd_options: Vec<String> = reqs
+        .as_ref()
+        .map(|r| r.options.clone())
+        .map(|r| get_cmd_options(r))
+        .unwrap_or_default();
 
     let inventories: Vec<String> = reqs
         .as_ref()
@@ -228,8 +266,9 @@ roles_path = ./roles
         .unwrap_or_else(|| vec![]);
 
     let authed_client = client.get_authed().await;
+    let mut nsjail_extra_mounts = vec![];
     if let Some(r) = reqs {
-        create_file_resources(
+        nsjail_extra_mounts = create_file_resources(
             &job.id,
             &job.workspace_id,
             job_dir,
@@ -261,18 +300,102 @@ roles_path = ./roles
         db,
     )
     .await;
-
-    if !*DISABLE_NSJAIL {
-        return Err(anyhow!("Ansible is not supported with nsjail, disable nsjail on your worker to run ansible playbooks").into());
-    }
+    let ansible_cfg_content = format!(
+        r#"
+[defaults]
+collections_path = ./
+roles_path = ./roles
+home={job_dir}/.ansible
+local_tmp={job_dir}/.ansible/tmp
+remote_tmp={job_dir}/.ansible/tmp
+"#
+    );
+    write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
 
     let mut reserved_variables = get_reserved_variables(job, &authed_client.token, db).await?;
     let additional_python_paths_folders = additional_python_paths.join(":");
-    reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+
+    if !*DISABLE_NSJAIL {
+        let shared_deps = additional_python_paths
+            .into_iter()
+            .map(|pp| {
+                format!(
+                    r#"
+mount {{
+    src: "{pp}"
+    dst: "{pp}"
+    is_bind: true
+    rw: false
+}}
+        "#
+                )
+            })
+            .join("\n");
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount)
+                .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
+                .replace("{FILE_RESOURCES}", nsjail_extra_mounts.join("\n").as_str())
+                .replace(
+                    "{ADDITIONAL_PYTHON_PATHS}",
+                    additional_python_paths_folders.as_str(),
+                ),
+        )?;
+    } else {
+        reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+    }
 
     let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"];
     cmd_args.extend(inventories.iter().map(|s| s.as_str()));
-    let child = {
+    cmd_args.extend(cmd_options.iter().map(|s| s.as_str()));
+
+    let child = if !*DISABLE_NSJAIL {
+        let wrapper = format!(
+            r#"set -eou pipefail
+{0} "$@"
+if [ -f "result" ]; then
+    cat result > result_nsjail_mount.json
+fi
+if [ -f "result.json" ]; then
+    cat result.json > result_nsjail_mount.json
+fi
+"#,
+            ANSIBLE_PLAYBOOK_PATH.as_str()
+        );
+
+        let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
+
+        file.metadata()?.permissions().set_mode(0o777);
+        // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("BASE_URL", base_internal_url)
+            .args(
+                vec![
+                    "--config",
+                    "run.config.proto",
+                    "--",
+                    BIN_BASH.as_str(),
+                    "/tmp/wrapper.sh",
+                ]
+                .into_iter()
+                .chain(cmd_args),
+            )
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
+    } else {
         let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str());
         ansible_cmd
             .current_dir(job_dir)
@@ -306,6 +429,54 @@ roles_path = ./roles
     read_and_check_result(job_dir).await
 }
 
+fn get_cmd_options(r: windmill_parser_yaml::AnsiblePlaybookOptions) -> Vec<String> {
+    let mut ret = vec![];
+
+    if let Some(v) = r.verbosity {
+        if v.chars().all(|c| c == 'v') && v.len() <= 6 {
+            ret.push(format!("-{}", v));
+        }
+    }
+
+    if let Some(o) = r.timeout {
+        ret.push("--timeout".to_string());
+        ret.push(o.to_string());
+    }
+
+    if let Some(o) = r.forks {
+        ret.push("--forks".to_string());
+        ret.push(o.to_string());
+    }
+
+    if r.flush_cache.is_some() {
+        ret.push("--flush-cache".to_string());
+    }
+
+    if r.force_handlers.is_some() {
+        ret.push("--force-handlers".to_string());
+    }
+
+    ret
+}
+
+fn define_nsjail_mount(job_dir: &str, path: &PathBuf) -> anyhow::Result<String> {
+    Ok(format!(
+        r#"
+mount {{
+    src: "{0}/{1}"
+    dst: "/tmp/{1}"
+    is_bind: true
+    rw: false
+    mandatory: false
+}}
+        "#,
+        job_dir,
+        path.strip_prefix(job_dir)?
+            .to_str()
+            .ok_or(anyhow!("Invalid path."))?
+    ))
+}
+
 async fn create_file_resources(
     job_id: &Uuid,
     w_id: &str,
@@ -314,8 +485,9 @@ async fn create_file_resources(
     r: &AnsibleRequirements,
     client: &crate::AuthedClient,
     db: &sqlx::Pool<sqlx::Postgres>,
-) -> error::Result<()> {
+) -> error::Result<Vec<String>> {
     let mut logs = String::new();
+    let mut nsjail_mounts: Vec<String> = vec![];
 
     for inventory in &r.inventories {
         let content;
@@ -336,9 +508,13 @@ async fn create_file_resources(
 
             content = serde_json::from_str(o.get())
                 .map_err(|e| anyhow!("Failed to parse inventory arg: {}", e))?;
+
+            if content == serde_json::value::Value::Null {
+                Err(anyhow!("The inventory argument was left empty. If you do not wish to specify an inventory for this script, remove the `inventory:` section from the yaml."))?;
+            }
         }
 
-        write_file_at_user_defined_location(
+        let validated_path = write_file_at_user_defined_location(
             job_dir,
             &inventory.name,
             content
@@ -349,6 +525,12 @@ async fn create_file_resources(
                 ))?,
         )
         .map_err(|e| anyhow!("Couldn't write inventory: {}", e))?;
+
+        nsjail_mounts.push(
+            define_nsjail_mount(job_dir, &validated_path)
+                .map_err(|e| anyhow!("Inventory path (a.k.a. `name`) is invalid: {}", e))?,
+        );
+
         logs.push_str(&format!("\nCreated inventory `{}`", inventory.name));
     }
 
@@ -360,7 +542,7 @@ async fn create_file_resources(
             )
             .await?;
         let path = file_res.target_path.clone();
-        write_file_at_user_defined_location(
+        let validated_path = write_file_at_user_defined_location(
             job_dir,
             path.as_str(),
             r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!(
@@ -369,6 +551,12 @@ async fn create_file_resources(
             ))?,
         )
         .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
+
+        nsjail_mounts.push(
+            define_nsjail_mount(job_dir, &validated_path)
+                .map_err(|e| anyhow!("File resource path is invalid: {}", e))?,
+        );
+
         logs.push_str(&format!(
             "\nCreated {} from {}",
             file_res.target_path, file_res.resource_path
@@ -376,5 +564,5 @@ async fn create_file_resources(
     }
     append_logs(job_id, w_id, logs, db).await;
 
-    Ok(())
+    Ok(nsjail_mounts)
 }

backend/windmill-worker/src/bash_executor.rs

@@ -12,7 +12,7 @@ use windmill_common::{
 use windmill_queue::{append_logs, CanceledBy};
 
 lazy_static::lazy_static! {
-    static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
+    pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
 }
 const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
 const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str =

backend/windmill-worker/src/bun_executor.rs

@@ -14,7 +14,8 @@ use crate::common::build_envs_map;
 use crate::{
     common::{
         create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
-        parse_npm_config, read_file_content, read_result, start_child_process, write_file_binary,
+        parse_npm_config, read_file, read_file_content, read_result, start_child_process,
+        write_file_binary,
     },
     AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
@@ -34,7 +35,7 @@ use windmill_common::variables;
 use windmill_common::{
     error::{self, Result},
     get_latest_hash_for_path,
-    jobs::QueuedJob,
+    jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
     scripts::ScriptLang,
     worker::{exists_in_cache, get_annotation, save_cache, write_file},
     DB,
@@ -749,6 +750,7 @@ pub async fn handle_bun_job(
     worker_name: &str,
     envs: HashMap<String, String>,
     shared_mount: &str,
+    new_args: &mut Option<HashMap<String, Box<RawValue>>>,
 ) -> error::Result<Box<RawValue>> {
     let mut annotation = windmill_common::worker::get_annotation(inner_content);
 
@@ -782,7 +784,16 @@ pub async fn handle_bun_job(
     if codebase.is_some() {
         annotation.nodejs_mode = true
     }
-    let main_override = get_main_override(job.args.as_ref());
+    let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) {
+        Some(main_override) => {
+            if main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
+                (None, true)
+            } else {
+                (Some(main_override), false)
+            }
+        }
+        None => (None, false),
+    };
 
     #[cfg(not(feature = "enterprise"))]
     if annotation.nodejs_mode || annotation.npm_mode {
@@ -937,18 +948,35 @@ pub async fn handle_bun_job(
         let args =
             windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())?
                 .args;
-        let dates = args
+
+        let pre_args = if apply_preprocessor {
+            Some(
+                windmill_parser_ts::parse_deno_signature(
+                    inner_content,
+                    true,
+                    Some("preprocessor".to_string()),
+                )?
+                .args,
+            )
+        } else {
+            None
+        };
+
+        let dates = pre_args
+            .as_ref()
+            .unwrap_or(&args)
             .iter()
-            .enumerate()
-            .filter_map(|(i, x)| {
+            .filter_map(|x| {
                 if matches!(x.typ, Typ::Datetime) {
-                    Some(i)
+                    Some(x.name.as_str())
                 } else {
                     None
                 }
             })
-            .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
-            .join("\n");
+            .map(|x| {
+                return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#);
+            })
+            .join("\n    ");
 
         let spread = args.into_iter().map(|x| x.name).join(",");
         // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
@@ -961,22 +989,47 @@ pub async fn handle_bun_job(
             "./main.ts"
         };
 
-        let wrapper_content: String = format!(
+        let preprocessor = if let Some(pre_args) = pre_args {
+            let pre_spread = pre_args.into_iter().map(|x| x.name).join(",");
+            format!(
+                r#"if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{
+        throw new Error("preprocessor function is missing");
+    }}
+    function preArgsObjToArr({{ {pre_spread} }}) {{
+        return [ {pre_spread} ];
+    }}
+    args = await Main.preprocessor(...preArgsObjToArr(args));
+    const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value);
+    await fs.writeFile('args.json', args_json, {{ encoding: 'utf8' }})"#
+            )
+        } else {
+            "".to_string()
+        };
+
+        let wrapper_content = format!(
             r#"
 import * as Main from "{main_import}";
 
 import * as fs from "fs/promises";
 
-const args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse)
-    .then(({{ {spread} }}) => [ {spread} ])
+let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse);
+
+function argsObjToArr({{ {spread} }}) {{
+    return [ {spread} ];
+}}
 
 BigInt.prototype.toJSON = function () {{
     return this.toString();
 }};
 
-{dates}
 async function run() {{
-    let res = await Main.{main_name}(...args);
+    {dates}
+    {preprocessor}
+    const argsArr = argsObjToArr(args);
+    if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{
+        throw new Error("{main_name} function is missing");
+    }}
+    let res = await Main.{main_name}(...argsArr);
     const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
     await fs.writeFile("result.json", res_json);
     process.exit(0);
@@ -1309,6 +1362,23 @@ try {{
         false,
     )
     .await?;
+
+    if apply_preprocessor {
+        let args = read_file(&format!("{job_dir}/args.json"))
+            .await
+            .map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while reading args from preprocessing: {e:#}"
+                ))
+            })?;
+        let args: HashMap<String, Box<RawValue>> =
+            serde_json::from_str(args.get()).map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while deserializing args from preprocessing: {e:#}"
+                ))
+            })?;
+        *new_args = Some(args.clone());
+    }
     read_result(job_dir).await
 }
 

backend/windmill-worker/src/common.rs

@@ -107,7 +107,7 @@ pub async fn create_args_and_out_file(
     job_dir: &str,
     db: &Pool<Postgres>,
 ) -> Result<(), Error> {
-    if let Some(args) = &job.args {
+    if let Some(args) = job.args.as_ref() {
         if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
             write_file(
                 job_dir,
@@ -380,8 +380,8 @@ pub async fn read_and_check_file(path: &str) -> error::Result<Box<RawValue>> {
 
     check_result_too_big(content.len())?;
 
-    let raw_value: Box<RawValue> = serde_json::from_str(&content)
-        .map_err(|e| anyhow!("{} is not valid json: {}", path, e))?;
+    let raw_value: Box<RawValue> =
+        serde_json::from_str(&content).map_err(|e| anyhow!("{} is not valid json: {}", path, e))?;
     Ok(raw_value)
 }
 
@@ -573,6 +573,28 @@ pub enum UpdateJobPollingExit {
     AlreadyCompleted,
 }
 
+pub async fn update_worker_ping_for_failed_init_script(
+    db: &DB,
+    worker_name: &str,
+    last_job_id: Uuid,
+) {
+    if let Err(e) = sqlx::query!(
+        "UPDATE worker_ping SET 
+        ping_at = now(), 
+        jobs_executed = 1, 
+        current_job_id = $1, 
+        current_job_workspace_id = 'admins' 
+        WHERE worker = $2",
+        last_job_id,
+        worker_name
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Error updating worker ping for failed init script: {e:?}");
+    }
+}
+
 pub async fn update_job_poller<F, Fut>(
     job_id: Uuid,
     db: &DB,

backend/windmill-worker/src/deno_executor.rs

@@ -8,13 +8,15 @@ use windmill_queue::{append_logs, CanceledBy};
 use crate::{
     common::{
         create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
-        parse_npm_config, read_result, start_child_process,
+        parse_npm_config, read_file, read_result, start_child_process,
     },
     AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
     NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
 };
 use tokio::{fs::File, io::AsyncReadExt, process::Command};
-use windmill_common::{error::Result, worker::write_file, BASE_URL};
+use windmill_common::{
+    error::Result, jobs::PREPROCESSOR_FAKE_ENTRYPOINT, worker::write_file, BASE_URL,
+};
 use windmill_common::{
     error::{self},
     jobs::QueuedJob,
@@ -170,12 +172,22 @@ pub async fn handle_deno_job(
     base_internal_url: &str,
     worker_name: &str,
     envs: HashMap<String, String>,
+    new_args: &mut Option<HashMap<String, Box<RawValue>>>,
 ) -> error::Result<Box<RawValue>> {
     // let mut start = Instant::now();
     let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
     append_logs(&job.id, &job.workspace_id, logs1, db).await;
 
-    let main_override = get_main_override(job.args.as_ref());
+    let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) {
+        Some(main_override) => {
+            if main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
+                (None, true)
+            } else {
+                (Some(main_override), false)
+            }
+        }
+        None => (None, false),
+    };
 
     write_file(job_dir, "main.ts", inner_content)?;
 
@@ -184,37 +196,83 @@ pub async fn handle_deno_job(
         let args =
             windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())?
                 .args;
-        let dates = args
+
+        let pre_args = if apply_preprocessor {
+            Some(
+                windmill_parser_ts::parse_deno_signature(
+                    inner_content,
+                    true,
+                    Some("preprocessor".to_string()),
+                )?
+                .args,
+            )
+        } else {
+            None
+        };
+
+        let dates = pre_args
+            .as_ref()
+            .unwrap_or(&args)
             .iter()
-            .enumerate()
-            .filter_map(|(i, x)| {
+            .filter_map(|x| {
                 if matches!(x.typ, Typ::Datetime) {
-                    Some(i)
+                    Some(x.name.as_str())
                 } else {
                     None
                 }
             })
-            .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
-            .join("\n");
+            .map(|x| {
+                return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#);
+            })
+            .join("\n    ");
 
         let spread = args.into_iter().map(|x| x.name).join(",");
         let main_name = main_override.unwrap_or("main".to_string());
         // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
+        let (preprocessor_import, preprocessor) = if let Some(pre_args) = pre_args {
+            let pre_spread = pre_args.into_iter().map(|x| x.name).join(",");
+            (
+                r#"import { preprocessor } from "./main.ts";"#.to_string(),
+                format!(
+                    r#"if (preprocessor === undefined || typeof preprocessor !== 'function') {{
+        throw new Error("preprocessor function is missing");
+    }}
+    function preArgsObjToArr({{ {pre_spread} }}) {{
+        return [ {pre_spread} ];
+    }}
+    args = await preprocessor(...preArgsObjToArr(args));
+    const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value);
+    await Deno.writeTextFile("args.json", args_json);"#
+                ),
+            )
+        } else {
+            ("".to_string(), "".to_string())
+        };
+
         let wrapper_content: String = format!(
             r#"
 import {{ {main_name} }} from "./main.ts";
+{preprocessor_import}
 
-const args = await Deno.readTextFile("args.json")
-    .then(JSON.parse)
-    .then(({{ {spread} }}) => [ {spread} ])
+let args = await Deno.readTextFile("args.json")
+    .then(JSON.parse);
+
+function argsObjToArr({{ {spread} }}) {{
+    return [ {spread} ];
+}}
 
 BigInt.prototype.toJSON = function () {{
     return this.toString();
 }};
 
-{dates}
 async function run() {{
-    let res: any = await {main_name}(...args);
+    {dates}
+    {preprocessor}
+    const argsArr = argsObjToArr(args);
+    if ({main_name} === undefined || typeof {main_name} !== 'function') {{
+        throw new Error("{main_name} function is missing");
+    }}
+    let res: any = await {main_name}(...argsArr);
     const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
     await Deno.writeTextFile("result.json", res_json);
     Deno.exit(0);
@@ -346,6 +404,22 @@ try {{
     {
         tracing::error!("failed to remove deno gen tmp cache dir: {}", e);
     }
+    if apply_preprocessor {
+        let args = read_file(&format!("{job_dir}/args.json"))
+            .await
+            .map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while reading args from preprocessing: {e:#}"
+                ))
+            })?;
+        let args: HashMap<String, Box<RawValue>> =
+            serde_json::from_str(args.get()).map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while deserializing args from preprocessing: {e:#}"
+                ))
+            })?;
+        *new_args = Some(args.clone());
+    }
     read_result(job_dir).await
 }
 

backend/windmill-worker/src/python_executor.rs

@@ -14,7 +14,7 @@ use uuid::Uuid;
 use windmill_common::ee::{get_license_plan, LicensePlan};
 use windmill_common::{
     error::{self, Error},
-    jobs::QueuedJob,
+    jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
     utils::calculate_hash,
     worker::{write_file, WORKER_CONFIG},
     DB,
@@ -56,7 +56,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
 use crate::{
     common::{
         create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
-        read_result, start_child_process,
+        read_file, read_result, start_child_process,
     },
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
     LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
@@ -246,6 +246,7 @@ pub async fn handle_python_job(
     shared_mount: &str,
     base_internal_url: &str,
     envs: HashMap<String, String>,
+    new_args: &mut Option<HashMap<String, Box<RawValue>>>,
 ) -> windmill_common::error::Result<Box<RawValue>> {
     let script_path = crate::common::use_flow_root_path(job.script_path());
     let additional_python_paths = handle_python_deps(
@@ -281,10 +282,39 @@ pub async fn handle_python_job(
         transforms,
         spread,
         main_name,
-    ) = prepare_wrapper(job_dir, inner_content, &script_path, job.args.as_ref()).await?;
+        pre_spread,
+    ) = prepare_wrapper(
+        job_dir,
+        inner_content,
+        &script_path,
+        job.args.as_ref(),
+        false,
+    )
+    .await?;
+
+    let apply_preprocessor = pre_spread.is_some();
 
     create_args_and_out_file(&client, job, job_dir, db).await?;
 
+    let preprocessor = if let Some(pre_spread) = pre_spread {
+        format!(
+            r#"if inner_script.preprocessor is None or not callable(inner_script.preprocessor):
+        raise ValueError("preprocessor function is missing")
+    else:
+        pre_args = {{}}
+        {pre_spread}
+        for k, v in list(pre_args.items()):
+            if v == '<function call>':
+                del pre_args[k]
+        kwargs = inner_script.preprocessor(**pre_args)
+        kwrags_json = res_to_json(kwargs)    
+        with open("args.json", 'w') as f:
+            f.write(kwrags_json)"#
+        )
+    } else {
+        "".to_string()
+    };
+
     let os_main_override = if let Some(main_override) = main_name.as_ref() {
         format!("os.environ[\"MAIN_OVERRIDE\"] = \"{main_override}\"\n")
     } else {
@@ -308,10 +338,6 @@ with open("args.json") as f:
     kwargs = json.load(f, strict=False)
 args = {{}}
 {transforms}
-{spread}
-for k, v in list(args.items()):
-    if v == '<function call>':
-        del args[k]
 
 def to_b_64(v: bytes):
     import base64
@@ -322,8 +348,7 @@ replace_nan = re.compile(r'(?:\bNaN\b|\\*\\u0000)')
 
 result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json")
 
-try:
-    res = inner_script.{main_override}(**args)
+def res_to_json(res):
     typ = type(res)
     if typ.__name__ == 'DataFrame':
         if typ.__module__ == 'pandas.core.frame':
@@ -336,7 +361,18 @@ try:
         for k, v in res.items():
             if type(v).__name__ == 'bytes':
                 res[k] = to_b_64(v)
-    res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
+    return re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
+
+try:
+    {preprocessor}
+    {spread}
+    for k, v in list(args.items()):
+        if v == '<function call>':
+            del args[k]
+    if inner_script.{main_override} is None or not callable(inner_script.{main_override}):
+        raise ValueError("{main_override} function is missing")
+    res = inner_script.{main_override}(**args)
+    res_json = res_to_json(res)
     with open(result_json, 'w') as f:
         f.write(res_json)
 except BaseException as e:
@@ -451,6 +487,24 @@ mount {{
         false,
     )
     .await?;
+
+    if apply_preprocessor {
+        let args = read_file(&format!("{job_dir}/args.json"))
+            .await
+            .map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while reading args from preprocessing: {e:#}"
+                ))
+            })?;
+        let args: HashMap<String, Box<RawValue>> =
+            serde_json::from_str(args.get()).map_err(|e| {
+                error::Error::InternalErr(format!(
+                    "error while deserializing args from preprocessing: {e:#}"
+                ))
+            })?;
+        *new_args = Some(args.clone());
+    }
+
     read_result(job_dir).await
 }
 
@@ -459,6 +513,7 @@ async fn prepare_wrapper(
     inner_content: &str,
     script_path: &str,
     args: Option<&Json<HashMap<String, Box<RawValue>>>>,
+    skip_preprocessor: bool,
 ) -> error::Result<(
     &'static str,
     &'static str,
@@ -469,8 +524,18 @@ async fn prepare_wrapper(
     String,
     String,
     Option<String>,
+    Option<String>,
 )> {
-    let main_override = get_main_override(args);
+    let (main_override, apply_preprocessor) = match get_main_override(args) {
+        Some(main_override) => {
+            if !skip_preprocessor && main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
+                (None, true)
+            } else {
+                (Some(main_override), false)
+            }
+        }
+        None => (None, false),
+    };
 
     let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
 
@@ -511,7 +576,20 @@ async fn prepare_wrapper(
     }
 
     let sig = windmill_parser_py::parse_python_signature(inner_content, main_override.clone())?;
-    let transforms = sig
+
+    let pre_sig = if apply_preprocessor {
+        Some(windmill_parser_py::parse_python_signature(
+            inner_content,
+            Some("preprocessor".to_string()),
+        )?)
+    } else {
+        None
+    };
+
+    // transforms should be applied based on the signature of the first function called
+    let init_sig = pre_sig.as_ref().unwrap_or(&sig);
+
+    let transforms = init_sig
         .args
         .iter()
         .map(|x| match x.typ {
@@ -539,7 +617,7 @@ async fn prepare_wrapper(
     } else {
         ""
     };
-    let import_base64 = if sig
+    let import_base64 = if init_sig
         .args
         .iter()
         .any(|x| x.typ == windmill_parser::Typ::Bytes)
@@ -548,7 +626,7 @@ async fn prepare_wrapper(
     } else {
         ""
     };
-    let import_datetime = if sig
+    let import_datetime = if init_sig
         .args
         .iter()
         .any(|x| x.typ == windmill_parser::Typ::Datetime)
@@ -569,12 +647,38 @@ async fn prepare_wrapper(
                 } else {
                     format!(
                         r#"args["{name}"] = kwargs.get("{name}")
-if args["{name}"] is None:
-    del args["{name}"]"#
+    if args["{name}"] is None:
+        del args["{name}"]"#
                     )
                 }
             })
-            .join("\n")
+            .join("\n    ")
+    };
+
+    let pre_spread = if let Some(pre_sig) = pre_sig {
+        let spread = if pre_sig.star_kwargs {
+            "pre_args = kwargs".to_string()
+        } else {
+            pre_sig
+                .args
+                .into_iter()
+                .map(|x| {
+                    let name = &x.name;
+                    if x.default.is_none() {
+                        format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")")
+                    } else {
+                        format!(
+                            r#"pre_args["{name}"] = kwargs.get("{name}")
+    if pre_args["{name}"] is None:
+        del pre_args["{name}"]"#
+                        )
+                    }
+                })
+                .join("\n    ")
+        };
+        Some(spread)
+    } else {
+        None
     };
 
     let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
@@ -589,6 +693,7 @@ if args["{name}"] is None:
         transforms,
         spread,
         main_override,
+        pre_spread,
     ))
 }
 
@@ -1082,7 +1187,8 @@ pub async fn start_worker(
         transforms,
         spread,
         _,
-    ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref()).await?;
+        _,
+    ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref(), true).await?;
 
     {
         let indented_transforms = transforms
@@ -1090,11 +1196,6 @@ pub async fn start_worker(
             .map(|x| format!("    {}", x))
             .collect::<Vec<String>>()
             .join("\n");
-        let indented_spread = spread
-            .lines()
-            .map(|x| format!("    {}", x))
-            .collect::<Vec<String>>()
-            .join("\n");
 
         let wrapper_content: String = format!(
             r#"
@@ -1122,7 +1223,7 @@ for line in sys.stdin:
     kwargs = json.loads(line, strict=False)
     args = {{}}
 {indented_transforms}
-{indented_spread}
+    {spread}
     for k, v in list(args.items()):
         if v == '<function call>':
             del args[k]

backend/windmill-worker/src/result_processor.rs

@@ -1,6 +1,6 @@
 use serde::Serialize;
 use sqlx::{types::Json, Pool, Postgres};
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use uuid::Uuid;
 
@@ -53,26 +53,49 @@ pub async fn process_result(
     cached_res_path: Option<String>,
     token: String,
     column_order: Option<Vec<String>>,
+    new_args: Option<HashMap<String, Box<RawValue>>>,
     db: &DB,
-) -> error::Result<()> {
+) -> error::Result<bool> {
     match result {
         Ok(r) => {
-            let job = if let Some(column_order) = column_order {
-                let mut job_with_column_order = (*job).clone();
-                match job_with_column_order.flow_status {
-                    Some(_) => {
-                        tracing::warn!("flow_status was expected to be none");
-                    }
-                    None => {
-                        job_with_column_order.flow_status =
-                            Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
-                                "_metadata": {
-                                    "column_order": column_order
-                                }
-                            }))));
+            let job = if column_order.is_some() || new_args.is_some() {
+                let mut updated_job = (*job).clone();
+                if let Some(column_order) = column_order {
+                    match updated_job.flow_status {
+                        Some(_) => {
+                            tracing::warn!("flow_status was expected to be none");
+                        }
+                        None => {
+                            updated_job.flow_status =
+                                Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
+                                    "_metadata": {
+                                        "column_order": column_order
+                                    }
+                                }))));
+                        }
                     }
                 }
-                Arc::new(job_with_column_order)
+                if let Some(new_args) = new_args {
+                    match updated_job.flow_status {
+                        Some(_) => {
+                            tracing::warn!("flow_status was expected to be none");
+                        }
+                        None => {
+                            // TODO save original args somewhere
+                            // if let Some(args) = updated_job.args.as_mut() {
+                            //     args.0.remove(ENTRYPOINT_OVERRIDE);
+                            // }
+                            updated_job.flow_status =
+                                Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
+                                    "_metadata": {
+                                        "preprocessed_args": true
+                                    }
+                                }))));
+                        }
+                    }
+                    updated_job.args = Some(Json(new_args));
+                }
+                Arc::new(updated_job)
             } else {
                 job
             };
@@ -88,6 +111,7 @@ pub async fn process_result(
                 token,
             )
             .await;
+            Ok(true)
         }
         Err(e) => {
             let error_value = match e {
@@ -129,9 +153,9 @@ pub async fn process_result(
                 token,
             )
             .await;
+            Ok(false)
         }
-    };
-    Ok(())
+    }
 }
 
 pub async fn handle_receive_completed_job<

backend/windmill-worker/src/worker.rs

@@ -84,10 +84,9 @@ use rand::Rng;
 
 use crate::{
     ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{
-        build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
-         NO_LOGS_AT_ALL, SLOW_LOGS,
-    }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_job_error, handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{
-        handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress,
+        build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, NO_LOGS_AT_ALL, SLOW_LOGS
+    }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{
+        handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step,
     }, worker_lockfiles::{
         handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
     }
@@ -195,6 +194,7 @@ pub async fn create_token_for_owner(
         exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp()
             as usize,
         job_id: Some(job_id.to_string()),
+        scopes: None,
     };
 
     let token = jsonwebtoken::encode(
@@ -1143,6 +1143,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     let job_completed_processor_is_done2 = job_completed_processor_is_done.clone();
     let same_worker_queue_size2 = same_worker_queue_size.clone();
+
+
     let send_result = tokio::spawn(
         (async move {
             let mut has_been_killed = false;
@@ -1178,6 +1180,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         if is_init_script_and_failure {
                             tracing::error!("init script errored, exiting");
                             killpill_tx2.send(()).unwrap_or_default();
+                            break;
                         }
                         if is_dependency_job && is_dedicated_worker {
                             tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
@@ -1695,8 +1698,10 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                     #[cfg(feature = "prometheus")]
                     let tag = job.tag.clone();
 
+                    let is_init_script: bool = job.tag.as_str() == INIT_SCRIPT_TAG;
                     let arc_job = Arc::new(job);
-                    if let Err(err) = handle_queued_job(
+                    
+                    match handle_queued_job(
                         arc_job.clone(),
                         db,
                         &authed_client,
@@ -1712,10 +1717,9 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         worker_flow_initial_transition_duration.clone(),
                         worker_code_execution_duration.clone(),
                     )
-                    .await
-                    {
-                        let is_init_script = arc_job.tag.as_str() == INIT_SCRIPT_TAG;
-                        handle_job_error(
+                    .await {
+                        Err(err) => {
+                            handle_job_error(
                             db,
                             &authed_client.get_authed().await,
                             arc_job.as_ref(),
@@ -1730,12 +1734,20 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                             (&job_completed_tx.0).clone(),
                         )
                         .await;
-                        if is_init_script {
-                            tracing::error!("failed to execute init_script, exiting");
-                            killpill_tx.send(()).unwrap();
+                        if is_init_script  {
+                            tracing::error!("init script job failed (in handler), exiting");
+                            update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await;
                             break;
                         }
-                    };
+                    },
+                    Ok(false) if is_init_script => {
+                            tracing::error!("init script job failed, exiting");
+                            update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await;
+                            break;
+                        
+                    }
+                    _ => {}
+                }
 
                     #[cfg(feature = "prometheus")]
                     if let Some(duration) = _timer.map(|x| x.stop_and_record()) {
@@ -1810,6 +1822,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
         };
     }
 
+    tracing::info!("worker {} exiting", worker_name);
     // #[cfg(feature = "benchmark")]
     // {
     //     println!("Writing benchmark file");
@@ -1824,10 +1837,14 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     drop(dedicated_workers);
 
-    for handle in dedicated_handles {
-        if let Err(e) = handle.await {
-            tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
+    let has_dedicated_workers = !dedicated_handles.is_empty();
+    if has_dedicated_workers {
+        for handle in dedicated_handles {
+            if let Err(e) = handle.await {
+                tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
+            }
         }
+        tracing::info!("all dedicated workers have exited");
     }
 
     drop(job_completed_tx);
@@ -1843,7 +1860,7 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     same_worker_tx: SameWorkerSender,
     worker_name: &str,
     rsmq: Option<R>,
-) -> error::Result<()> {
+) -> error::Result<bool> {
     if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() {
         let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
         let ehm = HashMap::new();
@@ -1889,8 +1906,10 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             .await
             .map_err(to_anyhow)?;
         tracing::info!("Creating initial job {uuid} from initial script script: {content}");
+        Ok(true)
+    } else {
+        Ok(false)
     }
-    Ok(())
 }
 
 // async fn process_result<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
@@ -2041,7 +2060,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     worker_code_execution_metric: &mut f32,
     _worker_flow_initial_transition_duration: Option<Histo>,
     _worker_code_execution_duration: Option<Histo>,
-) -> windmill_common::error::Result<()> {
+) -> windmill_common::error::Result<bool> {
     if job.canceled {
         return Err(Error::JsonErr(canceled_job_to_result(&job)));
     }
@@ -2050,7 +2069,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     }
 
     #[cfg(any(not(feature = "enterprise"), feature = "sqlx"))]
-    if job.created_by.starts_with("email-trigger-") {
+    if job.created_by.starts_with("email-") {
         let daily_count = sqlx::query!(
             "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1"
         ).fetch_optional(db).await?.map(|x| serde_json::from_value::<i64>(x.value).unwrap_or(1));
@@ -2087,7 +2106,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         )
         .await?;
 
-        r
+        Some(r)
     } else {
         if let Some(parent_job) = job.parent_job {
             if let Err(e) = sqlx::query_scalar!(
@@ -2137,9 +2156,13 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             )
             .fetch_one(db)
             .await
-            .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?
+            .map_err(|e| Error::InternalErr(format!("Fetching script path from queue for caching purposes: {e:#}")))?
             .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?;
-            let step = step.unwrap_or(-1);
+            let step = match step.unwrap() {
+                Step::Step(i) => i.to_string(),
+                Step::PreprocessorStep => "preprocessor".to_string(),
+                Step::FailureStep => "failure".to_string(),
+            };
             Some(format!(
                 "{flow_path}/cache/{version_hash}/{step}/{args_hash}"
             ))
@@ -2182,12 +2205,10 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                 .await
                 .expect("send job completed");
 
-            return Ok(());
+            return Ok(true);
         }
     };
     if job.is_flow() {
-        #[cfg(feature = "prometheus")]
-        let timer = _worker_flow_initial_transition_duration.map(|x| x.start_timer());
         handle_flow(
             job,
             db,
@@ -2199,8 +2220,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             job_completed_tx.0.clone(),
         )
         .await?;
-        #[cfg(feature = "prometheus")]
-        timer.map(|x| x.stop_and_record());
+        Ok(true)
     } else {
         let mut logs = "".to_string();
         let mut mem_peak: i32 = 0;
@@ -2239,6 +2259,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
         append_logs(&job.id, &job.workspace_id, logs, db).await;
 
         let mut column_order: Option<Vec<String>> = None;
+        let mut new_args: Option<HashMap<String, Box<RawValue>>> = None;
         let result = match job.job_kind {
             JobKind::Dependencies => {
                 handle_dependency_job(
@@ -2306,6 +2327,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                     base_internal_url,
                     worker_name,
                     &mut column_order,
+                    &mut new_args,
                 )
                 .await;
                 *worker_code_execution_metric += metric_timer.elapsed().as_secs_f32();
@@ -2317,14 +2339,14 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
 
         //it's a test job, no need to update the db
         if job.as_ref().workspace_id == "" {
-            return Ok(());
+            return Ok(true);
         }
 
         if result
             .as_ref()
             .is_err_and(|err| matches!(err, &Error::AlreadyCompleted(_)))
         {
-            return Ok(());
+            return Ok(false);
         }
 
         process_result(
@@ -2337,11 +2359,11 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             cached_res_path,
             client.get_token().await,
             column_order,
+            new_args,
             db,
         )
-        .await?;
-    };
-    Ok(())
+        .await
+    }
 }
 
 
@@ -2432,7 +2454,7 @@ pub async fn get_script_content_by_hash(
             Option<String>,
             Option<ScriptLang>,
             Option<Vec<String>>,
-            Option<bool>,
+            Option<bool>
         ),
     >(
         "SELECT content, lock, language, envs, codebase LIKE '%.tar' as codebase FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -2473,6 +2495,7 @@ async fn handle_code_execution_job(
     base_internal_url: &str,
     worker_name: &str,
     column_order: &mut Option<Vec<String>>,
+    new_args: &mut Option<HashMap<String, Box<RawValue>>>,
 ) -> error::Result<Box<RawValue>> {
     let ContentReqLangEnvs {
         content: inner_content,
@@ -2705,6 +2728,7 @@ mount {{
                 &shared_mount,
                 base_internal_url,
                 envs,
+                new_args,
             )
             .await
         }
@@ -2721,6 +2745,7 @@ mount {{
                 base_internal_url,
                 worker_name,
                 envs,
+                new_args,
             )
             .await
         }
@@ -2739,6 +2764,7 @@ mount {{
                 worker_name,
                 envs,
                 &shared_mount,
+                new_args,
             )
             .await
         }
@@ -2838,6 +2864,7 @@ mount {{
                 db,
                 client,
                 &inner_content,
+                &shared_mount,
                 base_internal_url,
                 envs,
             ).await

backend/windmill-worker/src/worker_flow.rs

@@ -37,7 +37,7 @@ use windmill_common::flow_status::{
 use windmill_common::flows::add_virtual_items_if_necessary;
 use windmill_common::jobs::{
     script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob,
-    RawCode,
+    RawCode, ENTRYPOINT_OVERRIDE,
 };
 use windmill_common::worker::to_raw_value;
 use windmill_common::{
@@ -225,14 +225,22 @@ pub async fn update_flow_status_after_job_completion_internal<
                 )))
             })?;
 
-        let module_index = usize::try_from(old_status.step).ok();
+        let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len());
 
-        let module_status = module_index
-            .and_then(|i| old_status.modules.get(i))
-            .unwrap_or(&old_status.failure_module.module_status);
+        let module_status = match module_step {
+            Step::PreprocessorStep => old_status
+                .preprocessor_module
+                .as_ref()
+                .ok_or_else(|| Error::InternalErr(format!("preprocessor module not found")))?,
+            Step::FailureStep => &old_status.failure_module.module_status,
+            Step::Step(i) => old_status
+                .modules
+                .get(i as usize)
+                .ok_or_else(|| Error::InternalErr(format!("module {i} not found")))?,
+        };
 
         // tracing::debug!(
-        //     "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} "
+        //     "UPDATE FLOW STATUS 2: {module_step:#?} {module_status:#?} {old_status:#?} "
         // );
 
         let (is_loop, skip_loop_failures, parallelism) = if matches!(
@@ -262,14 +270,14 @@ pub async fn update_flow_status_after_job_completion_internal<
             let flow_job = get_queued_job(&flow, w_id, db)
                 .await?
                 .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
-            let module = get_module(&flow_job, module_index);
+            let module = get_module(&flow_job, &module_step);
 
             if module.is_some_and(|x| x.is_flow()) {
                 (false, false, false)
             } else {
                 (true, se, false)
             }
-        } else if is_failure_step {
+        } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
             (false, false, false)
         } else {
             let r = sqlx::query_as::<_, SkipIfStopped>(
@@ -346,6 +354,28 @@ pub async fn update_flow_status_after_job_completion_internal<
 
         let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
 
+        if matches!(module_step, Step::PreprocessorStep) {
+            sqlx::query!(
+                "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
+                job_id_for_status,
+                flow
+            ).execute(&mut tx).await.map_err(|e| {
+                Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}"))
+            })?;
+
+            sqlx::query!(
+                r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#,
+                job_id_for_status
+            )
+            .execute(&mut tx)
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!(
+                    "error while deleting args of preprocessing step: {e:#}"
+                ))
+            })?;
+        }
+
         let (inc_step_counter, new_status) = match module_status {
             FlowStatusModule::InProgress {
                 iterator,
@@ -713,6 +743,21 @@ pub async fn update_flow_status_after_job_completion_internal<
                         "error while setting flow status in failure step: {e:#}"
                     ))
                 })?;
+            } else if matches!(module_step, Step::PreprocessorStep) {
+                sqlx::query!(
+                    "UPDATE queue
+                    SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)
+                    WHERE id = $2",
+                    json!(new_status),
+                    flow
+                )
+                .execute(&mut tx)
+                .await
+                .map_err(|e| {
+                    Error::InternalErr(format!(
+                        "error while setting flow status in preprocessing step: {e:#}"
+                    ))
+                })?;
             } else {
                 sqlx::query!(
                     "UPDATE queue
@@ -824,8 +869,7 @@ pub async fn update_flow_status_after_job_completion_internal<
             .unwrap_or_else(|| "none".to_string());
         tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
 
-        let module = get_module(&flow_job, module_index);
-
+        let module = get_module(&flow_job, &module_step);
         // tracing::error!(
         //     "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled
         // );
@@ -1115,17 +1159,13 @@ async fn retrieve_flow_jobs_results(
     Ok(to_raw_value(&results))
 }
 
-fn get_module(flow_job: &QueuedJob, module_index: Option<usize>) -> Option<FlowModule> {
+fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option<FlowModule> {
     let raw_flow = flow_job.parse_raw_flow();
     if let Some(raw_flow) = raw_flow {
-        if let Some(i) = module_index {
-            if let Some(module) = raw_flow.modules.get(i) {
-                Some(module.clone())
-            } else {
-                raw_flow.failure_module.map(|x| *x.clone())
-            }
-        } else {
-            None
+        match module_step {
+            Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()),
+            Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()),
+            Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()),
         }
     } else {
         None
@@ -1293,56 +1333,86 @@ pub async fn update_flow_status_in_progress(
     w_id: &str,
     flow: Uuid,
     job_in_progress: Uuid,
-) -> error::Result<Option<i32>> {
+) -> error::Result<Step> {
     let step = get_step_of_flow_status(db, flow).await?;
-    if let Step::Step(step) = step {
-        sqlx::query(&format!(
-            "UPDATE queue
-                SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
-                WHERE id = $3 AND workspace_id = $4",
-        ))
-        .bind(json!(job_in_progress.to_string()))
-        .bind(json!("InProgress"))
-        .bind(flow)
-        .bind(w_id)
-        .execute(db)
-        .await?;
-        Ok(Some(step))
-    } else {
-        sqlx::query(&format!(
-            "UPDATE queue
-                SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2)
-                WHERE id = $3 AND workspace_id = $4",
-        ))
-        .bind(json!(job_in_progress.to_string()))
-        .bind(json!("InProgress"))
-        .bind(flow)
-        .bind(w_id)
-        .execute(db)
-        .await?;
-        Ok(None)
+    match step {
+        Step::Step(step) => {
+            sqlx::query(&format!(
+                "UPDATE queue
+                    SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
+                    WHERE id = $3 AND workspace_id = $4",
+            ))
+            .bind(json!(job_in_progress.to_string()))
+            .bind(json!("InProgress"))
+            .bind(flow)
+            .bind(w_id)
+            .execute(db)
+            .await?;
+        }
+        Step::PreprocessorStep => {
+            sqlx::query(&format!(
+                "UPDATE queue
+                    SET flow_status = jsonb_set(jsonb_set(flow_status, '{{preprocessor_module, job}}', $1), '{{preprocessor_module, type}}', $2)
+                    WHERE id = $3 AND workspace_id = $4",
+            ))
+            .bind(json!(job_in_progress.to_string()))
+            .bind(json!("InProgress"))
+            .bind(flow)
+            .bind(w_id)
+            .execute(db)
+            .await?;
+        }
+        Step::FailureStep => {
+            sqlx::query(&format!(
+                "UPDATE queue
+                    SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2)
+                    WHERE id = $3 AND workspace_id = $4",
+            ))
+            .bind(json!(job_in_progress.to_string()))
+            .bind(json!("InProgress"))
+            .bind(flow)
+            .bind(w_id)
+            .execute(db)
+            .await?;
+        }
+    }
+
+    Ok(step)
+}
+
+#[derive(Debug)]
+pub enum Step {
+    Step(usize),
+    PreprocessorStep,
+    FailureStep,
+}
+
+impl Step {
+    fn from_i32_and_len(step: i32, len: usize) -> Self {
+        if step < 0 {
+            Step::PreprocessorStep
+        } else if (step as usize) < len {
+            Step::Step(step as usize)
+        } else {
+            Step::FailureStep
+        }
     }
 }
 
-pub enum Step {
-    Step(i32),
-    FailureStep,
-}
 #[instrument(level = "trace", skip_all)]
 pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<Step> {
     let r = sqlx::query!(
-        "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len  FROM queue WHERE id = $1",
+        "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
         id
     )
     .fetch_one(db)
     .await
     .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?;
-    if r.step < r.len {
-        Ok(Step::Step(r.step.ok_or_else(|| {
-            Error::InternalErr("step is null".to_string())
-        })?))
+
+    if let Some(step) = r.step {
+        Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize))
     } else {
-        Ok(Step::FailureStep)
+        Err(Error::InternalErr("step is null".to_string()))
     }
 }
 
@@ -1539,14 +1609,20 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         .unwrap_or_else(|| "none".to_string());
     tracing::info!(id = %flow_job.id, root_id = %job_root, "pushing next flow job");
 
-    let mut i = usize::try_from(status.step)
-        .with_context(|| format!("invalid module index {}", status.step))?;
+    let mut step = Step::from_i32_and_len(status.step, flow.modules.len());
 
-    let mut status_module: FlowStatusModule = status
-        .modules
-        .get(i)
-        .cloned()
-        .unwrap_or_else(|| status.failure_module.module_status.clone());
+    let mut status_module = match step {
+        Step::Step(i) => status
+            .modules
+            .get(i)
+            .cloned()
+            .unwrap_or_else(|| status.failure_module.module_status.clone()),
+        Step::PreprocessorStep => status
+            .preprocessor_module
+            .clone()
+            .unwrap_or_else(|| status.failure_module.module_status.clone()),
+        Step::FailureStep => status.failure_module.module_status.clone(),
+    };
 
     let fj: mappable_rc::Marc<QueuedJob> = flow_job.clone().into();
     let arc_flow_job_args: Marc<HashMap<String, Box<RawValue>>> = Marc::map(fj, |x| {
@@ -1584,7 +1660,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         return Ok(());
     }
 
-    if i == 0 {
+    if matches!(step, Step::Step(0)) {
         if !flow_job.is_flow_step && flow_job.schedule_path.is_some() {
             let no_flow_overlap = sqlx::query_scalar!(
                 "SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2",
@@ -1672,7 +1748,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
     let arc_last_job_result = if status_module.is_failure() {
         // if job is being retried, pass the result of its previous failure
         last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}"))))
-    } else if i == 0 {
+    } else if matches!(step, Step::Step(0)) || matches!(step, Step::PreprocessorStep) {
         // if it's the first job executed in the flow, pass the flow args
         Arc::new(to_raw_value(&flow_job.args))
     } else {
@@ -1929,17 +2005,25 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         }
     }
 
-    let mut module: &FlowModule = flow
-        .modules
-        .get(i)
-        .or_else(|| flow.failure_module.as_deref())
-        .with_context(|| format!("no module at index {}", status.step))?;
+    let mut module = match step {
+        Step::Step(i) => flow
+            .modules
+            .get(i)
+            .with_context(|| format!("no module at index {}", i))?,
+        Step::PreprocessorStep => flow
+            .preprocessor_module
+            .as_ref()
+            .with_context(|| format!("no preprocessor module"))?,
+        Step::FailureStep => flow
+            .failure_module
+            .as_deref()
+            .with_context(|| format!("no failure module"))?,
+    };
 
     let current_id = &module.id;
-    let mut previous_id = if i >= 1 {
-        flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap()
-    } else {
-        String::new()
+    let mut previous_id = match step {
+        Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(),
+        _ => String::new(),
     };
 
     // calculate sleep if any
@@ -1952,10 +2036,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         ) {
             None
         } else {
-            let sleep_input_transform = i
-                .checked_sub(1)
-                .and_then(|i| flow.modules.get(i))
-                .and_then(|m| m.sleep.clone());
+            let sleep_input_transform = if let Step::Step(i) = step {
+                i.checked_sub(1)
+                    .and_then(|i| flow.modules.get(i))
+                    .and_then(|m| m.sleep.clone())
+            } else {
+                None
+            };
 
             if let Some(it) = sleep_input_transform {
                 let json_value = match it {
@@ -2054,7 +2141,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                  * The failure module may also run again if it fails and the retry feature is used.
                  * In that case, `i` will index past `flow.modules`.  The above should handle that and
                  * re-run the failure module. */
-                i = flow.modules.len();
+                step = Step::FailureStep;
                 previous_id = current_id.clone();
                 module = flow
                     .failure_module
@@ -2121,6 +2208,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             } else {
                 Ok(Marc::new(HashMap::new()))
             }
+        } else if matches!(step, Step::PreprocessorStep) {
+            let mut hm = (*arc_flow_job_args).clone();
+            hm.insert(
+                ENTRYPOINT_OVERRIDE.to_string(),
+                to_raw_value(&"preprocessor"),
+            );
+            Ok(Marc::new(hm))
         } else {
             match &module.get_value() {
                 Ok(
@@ -2522,6 +2616,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}");
         }
     }
+
     let first_uuid = uuids[0];
     let new_status = match next_status {
         NextStatus::NextLoopIteration {
@@ -2610,36 +2705,52 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         }
     };
 
-    tracing::debug!("STATUS STEP: {:?} {i} {:#?}", status.step, new_status);
+    tracing::debug!("STATUS STEP: {:?} {step:?} {:#?}", status.step, new_status);
 
-    if i >= flow.modules.len() {
-        sqlx::query!(
-            "UPDATE queue
-            SET flow_status = JSONB_SET(
-                JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)
-            WHERE id = $3",
-            json!(FlowStatusModuleWParent {
-                parent_module: Some(current_id.clone()),
-                module_status: new_status
-            }),
-            json!(i),
-            flow_job.id
-        )
-        .execute(&mut tx)
-        .await?;
-    } else {
-        sqlx::query!(
-            "UPDATE queue
-            SET flow_status = JSONB_SET(
-                JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)
-            WHERE id = $4",
-            i as i32,
-            json!(new_status),
-            json!(i),
-            flow_job.id
-        )
-        .execute(&mut tx)
-        .await?;
+    match step {
+        Step::FailureStep => {
+            sqlx::query!(
+                "UPDATE queue
+                SET flow_status = JSONB_SET(
+                    JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)
+                WHERE id = $3",
+                json!(FlowStatusModuleWParent {
+                    parent_module: Some(current_id.clone()),
+                    module_status: new_status
+                }),
+                json!(flow.modules.len()),
+                flow_job.id
+            )
+            .execute(&mut tx)
+            .await?;
+        }
+        Step::PreprocessorStep => {
+            sqlx::query!(
+                "UPDATE queue
+                SET flow_status = JSONB_SET(
+                    JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)
+                WHERE id = $3",
+                json!(new_status),
+                json!(-1),
+                flow_job.id
+            )
+            .execute(&mut tx)
+            .await?;
+        }
+        Step::Step(i) => {
+            sqlx::query!(
+                "UPDATE queue
+                SET flow_status = JSONB_SET(
+                    JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)
+                WHERE id = $4",
+                i as i32,
+                json!(new_status),
+                json!(i),
+                flow_job.id
+            )
+            .execute(&mut tx)
+            .await?;
+        }
     };
 
     potentially_crash_for_testing();
@@ -2882,9 +2993,17 @@ async fn compute_next_flow_transform(
             concurrency_time_window_s,
             ..
         } => {
-            let path = path
-                .clone()
-                .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
+            let path = path.clone().or_else(|| {
+                if status
+                    .preprocessor_module
+                    .as_ref()
+                    .is_some_and(|x| x.id() == module.id)
+                {
+                    Some(format!("{}/preprocessor", flow_job.script_path()))
+                } else {
+                    Some(format!("{}/step-{}", flow_job.script_path(), status.step))
+                }
+            });
             let payload = raw_script_to_payload(
                 path,
                 content,
@@ -3011,6 +3130,7 @@ async fn compute_next_flow_transform(
                                 priority: None,
                                 early_return: None,
                                 concurrency_key: None,
+                                preprocessor_module: None,
                             };
                             ContinuePayload::ForloopJobs {
                                 n: itered.len(),
@@ -3108,6 +3228,7 @@ async fn compute_next_flow_transform(
                             priority: None,
                             early_return: None,
                             concurrency_key: None,
+                            preprocessor_module: None,
                         },
                         path: Some(format!(
                             "{}/branchone-{}",
@@ -3160,6 +3281,7 @@ async fn compute_next_flow_transform(
                                                     priority: None,
                                                     early_return: None,
                                                     concurrency_key: None,
+                                                    preprocessor_module: None,
                                                 },
                                                 path: Some(format!(
                                                     "{}/branchall-{}",
@@ -3234,6 +3356,7 @@ async fn compute_next_flow_transform(
                             priority: None,
                             early_return: None,
                             concurrency_key: None,
+                            preprocessor_module: None,
                         },
                         path: Some(format!(
                             "{}/branchall-{}",
@@ -3311,6 +3434,7 @@ async fn next_loop_iteration(
                         priority: None,
                         early_return: None,
                         concurrency_key: None,
+                        preprocessor_module: None,
                     },
                     path: inner_path,
                     restarted_from: None,
@@ -3548,7 +3672,11 @@ fn raw_script_to_payload(
 }
 
 fn flow_to_payload(path: &str, delete_after_use: &bool) -> JobPayloadWithTag {
-    let payload = JobPayload::Flow { path: path.to_string(), dedicated_worker: None };
+    let payload = JobPayload::Flow {
+        path: path.to_string(),
+        dedicated_worker: None,
+        apply_preprocessor: false,
+    };
     JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use, timeout: None }
 }
 
@@ -3562,7 +3690,7 @@ async fn script_to_payload(
 ) -> Result<JobPayloadWithTag, Error> {
     let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() {
         let (jp, tag, delete_after_use, script_timeout) =
-            script_path_to_payload(script_path, db, &flow_job.workspace_id).await?;
+            script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?;
         (
             jp,
             tag_override.to_owned().or(tag),
@@ -3595,6 +3723,7 @@ async fn script_to_payload(
                 language,
                 dedicated_worker,
                 priority,
+                apply_preprocessor: false,
             },
             tag_override.to_owned().or(tag),
             delete_after_use,

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.397.4";
+export const VERSION = "v1.399.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/.gitignore

@@ -1,2 +1 @@
 npm/
-gen/

cli/build.sh

@@ -1,2 +1,3 @@
+#!/bin/bash
 ./gen_wm_client.sh
-deno run -A dnt.ts
+deno run -A dnt.ts

cli/gen/core/ApiError.ts

@@ -0,0 +1,21 @@
+import type { ApiRequestOptions } from './ApiRequestOptions.ts';
+import type { ApiResult } from './ApiResult.ts';
+
+export class ApiError extends Error {
+	public readonly url: string;
+	public readonly status: number;
+	public readonly statusText: string;
+	public readonly body: unknown;
+	public readonly request: ApiRequestOptions;
+
+	constructor(request: ApiRequestOptions, response: ApiResult, message: string) {
+		super(message);
+
+		this.name = 'ApiError';
+		this.url = response.url;
+		this.status = response.status;
+		this.statusText = response.statusText;
+		this.body = response.body;
+		this.request = request;
+	}
+}

cli/gen/core/ApiRequestOptions.ts

@@ -0,0 +1,21 @@
+export type ApiRequestOptions<T = unknown> = {
+	readonly body?: any;
+	readonly cookies?: Record<string, unknown>;
+	readonly errors?: Record<number | string, string>;
+	readonly formData?: Record<string, unknown> | any[] | Blob | File;
+	readonly headers?: Record<string, unknown>;
+	readonly mediaType?: string;
+	readonly method:
+		| 'DELETE'
+		| 'GET'
+		| 'HEAD'
+		| 'OPTIONS'
+		| 'PATCH'
+		| 'POST'
+		| 'PUT';
+	readonly path?: Record<string, unknown>;
+	readonly query?: Record<string, unknown>;
+	readonly responseHeader?: string;
+	readonly responseTransformer?: (data: unknown) => Promise<T>;
+	readonly url: string;
+};

cli/gen/core/ApiResult.ts

@@ -0,0 +1,7 @@
+export type ApiResult<TData = any> = {
+	readonly body: TData;
+	readonly ok: boolean;
+	readonly status: number;
+	readonly statusText: string;
+	readonly url: string;
+};