feat(frontend): added a name blacklist for flow inputs

fix(frontend): correctly clear result when removing all options in a multi select (#2600 )
do not require being a super admin for default error handler
2023-11-09 15:14:18 +01:00 · 2023-11-09 14:25:49 +01:00 · 2023-11-09 12:07:14 +01:00 · 2023-11-09 12:01:38 +01:00 · 2023-11-09 11:58:45 +01:00 · 2023-11-09 11:56:12 +01:00
335 changed files with 12020 additions and 4134 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,189 @@
 # Changelog


+## [1.202.1](https://github.com/windmill-labs/windmill/compare/v1.202.0...v1.202.1) (2023-11-07)
+
+
+### Bug Fixes
+
+* remove the FOO ([f89a01f](https://github.com/windmill-labs/windmill/commit/f89a01ff2fb26224884e59599f72da0b83fa4a0f))
+
+## [1.202.0](https://github.com/windmill-labs/windmill/compare/v1.201.0...v1.202.0) (2023-11-07)
+
+
+### Features
+
+* add diffs when editing workspace script inside a flow ([#2581](https://github.com/windmill-labs/windmill/issues/2581)) ([e84e38d](https://github.com/windmill-labs/windmill/commit/e84e38d3bd4e3cb95fc71f91b3bd133740d82b05))
+* Add override all schedule handlers button ([#2579](https://github.com/windmill-labs/windmill/issues/2579)) ([f2bff84](https://github.com/windmill-labs/windmill/commit/f2bff8450223d29d3de8edd2e60d483f6ced5caa))
+* add support for flows in vscode extension ([#2585](https://github.com/windmill-labs/windmill/issues/2585)) ([8a7fe93](https://github.com/windmill-labs/windmill/commit/8a7fe93559209e7aa5427f5b6a8b9e03df9da406))
+* **frontend:** Rework variable table ([#2576](https://github.com/windmill-labs/windmill/issues/2576)) ([b040a89](https://github.com/windmill-labs/windmill/commit/b040a89b27f6dca41049e4bceeae4e3665c005ae))
+
+
+### Bug Fixes
+
+* add tag support for CLI ([0ede0f4](https://github.com/windmill-labs/windmill/commit/0ede0f4c972eb1b65dcb542ea6facf5ee2c74cfd))
+* add tag sync for cli ([6c12c6e](https://github.com/windmill-labs/windmill/commit/6c12c6e7846c8be3f843ca7a98b8bac6fac1d7e8))
+* **frontend:** add missing classes when tabs are in sidebar mode ([#2577](https://github.com/windmill-labs/windmill/issues/2577)) ([dd3594c](https://github.com/windmill-labs/windmill/commit/dd3594c5e5624b712564126f046fde4fc06c42ca))
+* **frontend:** escape wasn't removing the hash when closing a drawer ([#2583](https://github.com/windmill-labs/windmill/issues/2583)) ([0951431](https://github.com/windmill-labs/windmill/commit/0951431e419c127cc562158447012960feb6d3af))
+* handle graphql invalid response ([#2582](https://github.com/windmill-labs/windmill/issues/2582)) ([12e731b](https://github.com/windmill-labs/windmill/commit/12e731b5c03fa788c9f0d00a955b7b01b4c570a0))
+
+## [1.201.0](https://github.com/windmill-labs/windmill/compare/v1.200.0...v1.201.0) (2023-11-06)
+
+
+### Features
+
+* add new stats ([#2568](https://github.com/windmill-labs/windmill/issues/2568)) ([1ed52ab](https://github.com/windmill-labs/windmill/commit/1ed52ab4c96988847c74c0672497dce1dd24ff6d))
+* Add toggle to optionally mute error handler for cancelled jobs ([#2567](https://github.com/windmill-labs/windmill/issues/2567)) ([83f9ef3](https://github.com/windmill-labs/windmill/commit/83f9ef34e6f7bb207f3410cbe2cceca7b52a4dea))
+* **frontend:** Ag grid styling ([#2565](https://github.com/windmill-labs/windmill/issues/2565)) ([97c5fe8](https://github.com/windmill-labs/windmill/commit/97c5fe8985166c9dae063de7e9d122914b190a4e))
+* telemetry disclosure ([#2562](https://github.com/windmill-labs/windmill/issues/2562)) ([1bb8b60](https://github.com/windmill-labs/windmill/commit/1bb8b606ed2e8ade12d5278072cc2b57c8d3ca27))
+
+
+### Bug Fixes
+
+* add no changes popup when saving draft ([#2571](https://github.com/windmill-labs/windmill/issues/2571)) ([d3dbb18](https://github.com/windmill-labs/windmill/commit/d3dbb188156cfd98e6cb1348d40e8854c008559e))
+* apps diffs ([#2570](https://github.com/windmill-labs/windmill/issues/2570)) ([3ed7ae7](https://github.com/windmill-labs/windmill/commit/3ed7ae7ffa28cb8b8c4799034ae8f0c8822fd519))
+* flow diffs ([#2561](https://github.com/windmill-labs/windmill/issues/2561)) ([aa5b71c](https://github.com/windmill-labs/windmill/commit/aa5b71ca05429788078547b3249eb1c3cd375ccc))
+* **frontend:** fix label event issues ([#2574](https://github.com/windmill-labs/windmill/issues/2574)) ([8935d22](https://github.com/windmill-labs/windmill/commit/8935d2272fcd630ccb1ab70ba0fa334934640fcb))
+* improve dedicated workers ([141f45b](https://github.com/windmill-labs/windmill/commit/141f45bf95388c0547e84980096a99607f3dba2f))
+* minor bug fixes ([#2566](https://github.com/windmill-labs/windmill/issues/2566)) ([e195202](https://github.com/windmill-labs/windmill/commit/e19520295f41784aae66df0d686b22fec810d57b))
+
+## [1.200.0](https://github.com/windmill-labs/windmill/compare/v1.199.0...v1.200.0) (2023-11-04)
+
+
+### Features
+
+* improve drafts and diffs ([#2534](https://github.com/windmill-labs/windmill/issues/2534)) ([3bfc2c8](https://github.com/windmill-labs/windmill/commit/3bfc2c81d2d405c8ea12d68622bbf7175b3947db))
+
+
+### Bug Fixes
+
+* **frontend:** fix treeview ([#2552](https://github.com/windmill-labs/windmill/issues/2552)) ([02764b1](https://github.com/windmill-labs/windmill/commit/02764b1fad7e2f44f46b49cbe7500266e9cc2f8e))
+* return non-integer sleep error directly ([#2558](https://github.com/windmill-labs/windmill/issues/2558)) ([543fae7](https://github.com/windmill-labs/windmill/commit/543fae77a75b88a1199f8d3cbb0460257ed5db95))
+
+## [1.199.0](https://github.com/windmill-labs/windmill/compare/v1.198.0...v1.199.0) (2023-11-03)
+
+
+### Features
+
+* Schedule error handler improvements ([#2555](https://github.com/windmill-labs/windmill/issues/2555)) ([668c9da](https://github.com/windmill-labs/windmill/commit/668c9da6461997c1b7907111bbfd0eff5e0ec159))
+
+
+### Bug Fixes
+
+* fail on non-integer sleep value ([#2556](https://github.com/windmill-labs/windmill/issues/2556)) ([6f47b96](https://github.com/windmill-labs/windmill/commit/6f47b9600629ec24a4e265a0ccc9eee75458229f))
+
+## [1.198.0](https://github.com/windmill-labs/windmill/compare/v1.197.1...v1.198.0) (2023-11-03)
+
+
+### Features
+
+* **frontend:** fix table when seaching with hidden columns ([#2549](https://github.com/windmill-labs/windmill/issues/2549)) ([0aaffad](https://github.com/windmill-labs/windmill/commit/0aaffadf0ca6964539011733b5d4882fdd26588a))
+* **frontend:** update displayed path for treeview ([#2551](https://github.com/windmill-labs/windmill/issues/2551)) ([0349ba5](https://github.com/windmill-labs/windmill/commit/0349ba5d567f2b52ff8058f347e501598ce4c981))
+
+
+### Bug Fixes
+
+* **frontend:** fix mobile sidebar opacity ([#2554](https://github.com/windmill-labs/windmill/issues/2554)) ([e1e48cf](https://github.com/windmill-labs/windmill/commit/e1e48cfc5f3cb68aa4bbd18a1a2ad6f0a300c374))
+* make graph rendering uniform across all rem ([#2553](https://github.com/windmill-labs/windmill/issues/2553)) ([0d4fc6a](https://github.com/windmill-labs/windmill/commit/0d4fc6a0bbc9b8e20c20bf2646eab202f795b8bd))
+* make python imports work at any nesting level ([75a5766](https://github.com/windmill-labs/windmill/commit/75a5766f8bf14c3749aff56fb94a8c04b32de4b6))
+* make timeline fit for high number of iterations for flows ([37eac60](https://github.com/windmill-labs/windmill/commit/37eac608666d903504872ce815839a7493fe876a))
+* subflow with cache can not be considered simple ([54f0812](https://github.com/windmill-labs/windmill/commit/54f08122d2837b5ee3d283733e0da403b21fadf0))
+* support results[&lt;x&gt;] + export more metatada for scripts ([0f37439](https://github.com/windmill-labs/windmill/commit/0f37439877ab498a615746357d771703db47a6d2))
+
+## [1.197.1](https://github.com/windmill-labs/windmill/compare/v1.197.0...v1.197.1) (2023-11-02)
+
+
+### Bug Fixes
+
+* fix cli ([77e0e2e](https://github.com/windmill-labs/windmill/commit/77e0e2ebc1fbe00eec431bd5d20619b89e8b7511))
+* Slack error handler missing "slack" arg ([#2546](https://github.com/windmill-labs/windmill/issues/2546)) ([7ba2a6c](https://github.com/windmill-labs/windmill/commit/7ba2a6c4f111b980181034ef5181193996c19fc4))
+
+## [1.197.0](https://github.com/windmill-labs/windmill/compare/v1.196.0...v1.197.0) (2023-11-02)
+
+
+### Features
+
+* **frontend:** add treeview ([#2542](https://github.com/windmill-labs/windmill/issues/2542)) ([86a2ced](https://github.com/windmill-labs/windmill/commit/86a2ced605fbab27bd01984c846e467a2612102b))
+* **frontend:** fix sidebar ([#2544](https://github.com/windmill-labs/windmill/issues/2544)) ([b4f043d](https://github.com/windmill-labs/windmill/commit/b4f043d32dd4fefc104d0fca429f4b39a23e1166))
+* **frontend:** fix sidebar color ([#2541](https://github.com/windmill-labs/windmill/issues/2541)) ([30a9460](https://github.com/windmill-labs/windmill/commit/30a9460cca676ac8f8e585024a0927ca90252f17))
+
+
+### Bug Fixes
+
+* enable default tokenizer truncation ([#2537](https://github.com/windmill-labs/windmill/issues/2537)) ([29aabd3](https://github.com/windmill-labs/windmill/commit/29aabd3472f59a4b5a657e7b046d66183d5fa0ba))
+* fix powershell args passing ([b4d5c5a](https://github.com/windmill-labs/windmill/commit/b4d5c5add8b92db1094e46c347efded52aa0f389))
+* improve rendering of list with undefined heights ([9eec2e2](https://github.com/windmill-labs/windmill/commit/9eec2e2c3e0183520cc50c716342bf329145edbd))
+
+## [1.196.0](https://github.com/windmill-labs/windmill/compare/v1.195.0...v1.196.0) (2023-11-01)
+
+
+### Features
+
+* improve inputs handling for large list on apps ([270d871](https://github.com/windmill-labs/windmill/commit/270d871039c708b7cfa218e22650fc25b1ec841c))
+
+## [1.195.0](https://github.com/windmill-labs/windmill/compare/v1.194.0...v1.195.0) (2023-10-31)
+
+
+### Features
+
+* Ability to restart flow on loop/branchall iteration ([#2526](https://github.com/windmill-labs/windmill/issues/2526)) ([c31299b](https://github.com/windmill-labs/windmill/commit/c31299bed8110f53e31f69983f144aaa82d5560d))
+* **frontend:** chartjs wizard ([#2532](https://github.com/windmill-labs/windmill/issues/2532)) ([03dfe71](https://github.com/windmill-labs/windmill/commit/03dfe711c6292d8d9af78ed610fd2885ad62b8d7))
+* invalidate result cache on flow or script change ([cf9669c](https://github.com/windmill-labs/windmill/commit/cf9669c18de6091dbf5dafad0f6ffd6e17675ca4))
+
+
+### Bug Fixes
+
+* add on success events to triggers list ([1974012](https://github.com/windmill-labs/windmill/commit/1974012621f3a2112eace60ba5d68854d567c9c2))
+* fix quick search scripts ([b3d2213](https://github.com/windmill-labs/windmill/commit/b3d2213ccec73bf4a7f27242ac31eb0941b791ac))
+* Load schedule statistics in background ([#2530](https://github.com/windmill-labs/windmill/issues/2530)) ([c98ebf9](https://github.com/windmill-labs/windmill/commit/c98ebf92e5f99f63cc3754555b5867f71c09e1a9))
+* only load embeddings if in server mode ([c803631](https://github.com/windmill-labs/windmill/commit/c8036317d23bf81b474742223e903255dd8825e0))
+
+## [1.194.0](https://github.com/windmill-labs/windmill/compare/v1.193.0...v1.194.0) (2023-10-30)
+
+
+### Features
+
+* **frontend:** plotly wizard ([#2517](https://github.com/windmill-labs/windmill/issues/2517)) ([40adfdb](https://github.com/windmill-labs/windmill/commit/40adfdb7fd6e21d053e79c6db876a4a8c90509c3))
+* Restartable flows ([#2514](https://github.com/windmill-labs/windmill/issues/2514)) ([76a736a](https://github.com/windmill-labs/windmill/commit/76a736aee67517807d86e1c8c3961af907fc919c))
+
+
+### Bug Fixes
+
+* assign column length of table actions based on number of actions ([0c672e7](https://github.com/windmill-labs/windmill/commit/0c672e7e18b52d4615dcf473bf1424dfe685cc9d))
+* fix table reactivity ([01560db](https://github.com/windmill-labs/windmill/commit/01560dbdaf2dc30c16e9182b5a3353d13c927827))
+* **frontend:** fix ai gen ([#2518](https://github.com/windmill-labs/windmill/issues/2518)) ([6f4fb76](https://github.com/windmill-labs/windmill/commit/6f4fb7668cd09d2e5fdca8977718fd6b87883a27))
+* make modal and drawer button hiddable as config ([2e55af5](https://github.com/windmill-labs/windmill/commit/2e55af50c70b9d1118e9a63f119ac31e5a574e51))
+* workspaced embeddings for resource types ([#2525](https://github.com/windmill-labs/windmill/issues/2525)) ([302649f](https://github.com/windmill-labs/windmill/commit/302649faa858233ea62073a2460e1586db67249f))
+
+## [1.193.0](https://github.com/windmill-labs/windmill/compare/v1.192.0...v1.193.0) (2023-10-28)
+
+
+### Features
+
+* refactor metrics and add performance debug metrics ([#2520](https://github.com/windmill-labs/windmill/issues/2520)) ([b888348](https://github.com/windmill-labs/windmill/commit/b8883481f46f1317748bb1884e0f0d287e8ae7fa))
+
+
+### Bug Fixes
+
+* **frontend:** add a disabled prop to text inputs ([#2512](https://github.com/windmill-labs/windmill/issues/2512)) ([7164de8](https://github.com/windmill-labs/windmill/commit/7164de81b02376f61eac965f08d75ab8b790e0ee))
+* **frontend:** fix insert new item ([#2519](https://github.com/windmill-labs/windmill/issues/2519)) ([c4383cf](https://github.com/windmill-labs/windmill/commit/c4383cfe740220f4674c93beeae9eb74397f7aff))
+* **frontend:** increased size limit for images ([#2510](https://github.com/windmill-labs/windmill/issues/2510)) ([aaa2657](https://github.com/windmill-labs/windmill/commit/aaa26579dc5e24f46e554a42c0121fb6a04d58f3))
+* move keep job directories and expose debug metrics to instance settings UI ([55ceca1](https://github.com/windmill-labs/windmill/commit/55ceca19131ac6dfb190f8818b18b46ca329babc))
+* prometheus metrics are an instance settings ([ea28163](https://github.com/windmill-labs/windmill/commit/ea28163865a22174dc1b92242a24989a1a47af21))
+
+## [1.192.0](https://github.com/windmill-labs/windmill/compare/v1.191.0...v1.192.0) (2023-10-25)
+
+
+### Features
+
+* **frontend:** add display borders configuration to list component ([#2508](https://github.com/windmill-labs/windmill/issues/2508)) ([dc54829](https://github.com/windmill-labs/windmill/commit/dc548292ac05f92116d2d65863da3a52e0cfe027))
+
+
+### Bug Fixes
+
+* do not share http_client in js_eval runtime ([402193c](https://github.com/windmill-labs/windmill/commit/402193cef9eff0ca03f5bd854d29f95774c4b73e))
+* fix global instance dynamic css ([8efe0ca](https://github.com/windmill-labs/windmill/commit/8efe0cadacaae894cf93a3a569e3d0b8e79c7d14))
+
 ## [1.191.0](https://github.com/windmill-labs/windmill/compare/v1.190.3...v1.191.0) (2023-10-24)


--- a/3
+++ b/3
@@ -3,4 +3,5 @@
        reverse_proxy /ws/* http://lsp:3001
        # reverse_proxy /ws_mp/* http://multiplayer:3002
        reverse_proxy /* http://windmill_server:8000
-}
+        # tls /certs/cert.pem /certs/key.pem
+}
--- a/6
+++ b/6
@@ -96,8 +96,8 @@ SHELL ["/bin/bash", "-c"]
 RUN apt update -y
 RUN apt install -y unzip curl

-RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.3/deno-linux-arm64.zip -o deno.zip || true
-RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.3/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.38.0/deno-linux-arm64.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.38.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true

 RUN unzip deno.zip && rm deno.zip

@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno

 COPY --from=nsjail /nsjail/nsjail /bin/nsjail

-COPY --from=oven/bun:1.0.6 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.8 /usr/local/bin/bun /usr/bin/bun

 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
--- a/README.md
+++ b/README.md
@@ -229,8 +229,8 @@ We publish helm charts at:

 ### Run from binaries

-Each release includes the corresponding binaries for x86_64. You can simply download the
-latest `windmill` binary using the following set of bash commands.
+Each release includes the corresponding binaries for x86_64. You can simply
+download the latest `windmill` binary using the following set of bash commands.

 ```bash
 BINARY_NAME='windmill-amd64' # or windmill-ee-amd64 for the enterprise edition
@@ -242,7 +242,10 @@ wget "$ARTIFACT_URL" -O windmill

 ### OAuth, SSO & SMTP

-Windmill Community Edition allows to configure the OAuth, SSO (including Google Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin settings. Do note that there is a limit of 50 SSO users on the community edition.
+Windmill Community Edition allows to configure the OAuth, SSO (including Google
+Workspace SSO, Microsoft/Azure and Okta) directly from the UI in the superadmin
+settings. Do note that there is a limit of 50 SSO users on the community
+edition.

 ### Commercial license

@@ -318,9 +321,9 @@ it being synced automatically everyday.
 | INSTANCE_EVENTS_WEBHOOK                       | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
 | GLOBAL_CACHE_INTERVAL                         | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
 | WORKER_TAGS                                   | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
-| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path>                                                                                                            | Worker                |
+| DEDICATED_WORKER                              | None                                       | Unique script to run on that worker. Has to be in the form of `<workspace>:<script_path>`                                                                                                            | Worker                |
 | CUSTOM_TAGS                                   | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
-| JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
+| JOB_RETENTION_SECS                            | 60*60*24\*60 //60 days                     | **Overriden by the instance settings UI** The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                         |
 | WAIT_RESULT_FAST_POLL_INTERVAL_MS             | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
 | WAIT_RESULT_SLOW_POLL_INTERVAL_MS             | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
 | WAIT_RESULT_FAST_POLL_DURATION_SECS           | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
--- a/backend/.sqlx/query-061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168.json
+++ b/backend/.sqlx/query-061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168.json
@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow_status FROM completed_job WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "flow_status",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168"
+}
--- a/backend/.sqlx/query-08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468.json
+++ b/backend/.sqlx/query-08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468.json
@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT login_type, COUNT(*) FROM password GROUP BY login_type",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "login_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468"
+}
--- a/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json
+++ b/backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json
@@ -42,7 +42,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json
+++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json
@@ -77,6 +77,11 @@
        "ordinal": 14,
        "name": "error_handler_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 15,
+        "name": "error_handler_muted_on_cancel",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -99,7 +104,8 @@
      true,
      true,
      false,
-      true
+      true,
+      false
    ]
  },
  "hash": "1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597"
--- a/backend/.sqlx/query-1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab.json
+++ b/backend/.sqlx/query-1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab"
+}
--- a/backend/.sqlx/query-2157813137b383411227b06814143df2d2187d1d43b84f67d22c0a2365b42ada.json
+++ b/backend/.sqlx/query-2157813137b383411227b06814143df2d2187d1d43b84f67d22c0a2365b42ada.json
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10 WHERE path = $11 AND workspace_id = $12 RETURNING *",
+  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11 \n        WHERE path  = $12 AND workspace_id = $13 RETURNING *",
  "describe": {
    "columns": [
      {
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -116,6 +121,7 @@
        "Varchar",
        "Int4",
        "Json",
+        "Bool",
        "Text",
        "Text"
      ]
@@ -140,8 +146,9 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
-  "hash": "0be97088e8ca1e8be9bf4ea45f830cceedd598a3221c522a1f9026d2d48a0ffb"
+  "hash": "2157813137b383411227b06814143df2d2187d1d43b84f67d22c0a2365b42ada"
 }
--- a/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json
+++ b/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json
@@ -67,7 +67,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-47a01c0924df7bc2695f8238425aa7eadf845e4156da8cf645582737e046589d.json
+++ b/backend/.sqlx/query-47a01c0924df7bc2695f8238425aa7eadf845e4156da8cf645582737e046589d.json
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *",
+  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING *",
  "describe": {
    "columns": [
      {
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -122,7 +127,8 @@
        "Json",
        "Varchar",
        "Int4",
-        "Json"
+        "Json",
+        "Bool"
      ]
    },
    "nullable": [
@@ -145,8 +151,9 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
-  "hash": "9b9afb9d5ef57b70de1207c4546b5547e579d978d899325d821a5aee1df1a9fc"
+  "hash": "47a01c0924df7bc2695f8238425aa7eadf845e4156da8cf645582737e046589d"
 }
--- a/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json
+++ b/backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json
@@ -28,7 +28,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-539d9ae486c7d1f5fb8c7278d698675b04359182b781b0cb0164402b798a7fef.json
+++ b/backend/.sqlx/query-539d9ae486c7d1f5fb8c7278d698675b04359182b781b0cb0164402b798a7fef.json
@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE schedule SET ws_error_handler_muted = $1, on_failure = $2, on_failure_extra_args = $3, on_failure_times = $4, on_failure_exact = $5 WHERE workspace_id = $6",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Varchar",
+        "Json",
+        "Int4",
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "539d9ae486c7d1f5fb8c7278d698675b04359182b781b0cb0164402b798a7fef"
+}
--- a/backend/.sqlx/query-542ebd3d6cd8522d117112ef9eb13ff95b93d67994823a4a84d37639eb17e0b9.json
+++ b/backend/.sqlx/query-542ebd3d6cd8522d117112ef9eb13ff95b93d67994823a4a84d37639eb17e0b9.json
@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE schedule SET on_recovery = $1, on_recovery_extra_args = $2, on_recovery_times = $3 WHERE workspace_id = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Json",
+        "Int4",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "542ebd3d6cd8522d117112ef9eb13ff95b93d67994823a4a84d37639eb17e0b9"
+}
--- a/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json
+++ b/backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json
@@ -77,6 +77,11 @@
        "ordinal": 14,
        "name": "error_handler_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 15,
+        "name": "error_handler_muted_on_cancel",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -99,7 +104,8 @@
      true,
      true,
      false,
-      true
+      true,
+      false
    ]
  },
  "hash": "5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e"
--- a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json
+++ b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json
@@ -60,7 +60,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-6a7eac3f0e5fbda17fff476d1a4a97059c12253caf8361257c4d26fc5b92b1b9.json
+++ b/backend/.sqlx/query-6a7eac3f0e5fbda17fff476d1a4a97059c12253caf8361257c4d26fc5b92b1b9.json
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group, wm_version FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
  "describe": {
    "columns": [
      {
@@ -42,6 +42,11 @@
        "ordinal": 7,
        "name": "worker_group",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "wm_version",
+        "type_info": "Varchar"
      }
    ],
    "parameters": {
@@ -58,8 +63,9 @@
      false,
      false,
      true,
+      false,
      false
    ]
  },
-  "hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
+  "hash": "6a7eac3f0e5fbda17fff476d1a4a97059c12253caf8361257c4d26fc5b92b1b9"
 }
--- a/backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json
+++ b/backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json
@@ -105,6 +105,11 @@
      },
      {
        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
        "name": "jobs",
        "type_info": "JsonArray"
      }
@@ -137,6 +142,7 @@
      true,
      true,
      true,
+      false,
      null
    ]
  },
--- a/backend/.sqlx/query-7b609ed87f974dfe887778746230b40b485658a75a1186c6ebb4c2de1b52a2fa.json
+++ b/backend/.sqlx/query-7b609ed87f974dfe887778746230b40b485658a75a1186c6ebb4c2de1b52a2fa.json
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
  "describe": {
    "columns": [],
    "parameters": {
@@ -10,10 +10,11 @@
        "Varchar",
        "TextArray",
        "Varchar",
+        "Varchar",
        "Varchar"
      ]
    },
    "nullable": []
  },
-  "hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
+  "hash": "7b609ed87f974dfe887778746230b40b485658a75a1186c6ebb4c2de1b52a2fa"
 }
--- a/backend/.sqlx/query-8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29.json
+++ b/backend/.sqlx/query-8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29.json
@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "canceled",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Uuid"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "8b221f0d08f3304364e56f5c4894fea42975b03d21904655d724a35163413d29"
-}
--- a/backend/.sqlx/query-911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90.json
+++ b/backend/.sqlx/query-911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -131,7 +136,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90"
--- a/backend/.sqlx/query-96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f.json
+++ b/backend/.sqlx/query-96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -132,7 +137,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "96dc1cd308f3c21cfb50b88048054dc03f93e261d25969d66aa48e9d0502960f"
--- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json
+++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json
@@ -40,7 +40,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json
+++ b/backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json
@@ -46,7 +46,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb.json
+++ b/backend/.sqlx/query-aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -130,7 +135,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "aa2800113a8a8805f47cdc1dd0f29d94c546fe531e7edd3e91da4978af5442fb"
--- a/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json
+++ b/backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json
@@ -42,7 +42,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3.json
+++ b/backend/.sqlx/query-c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -131,7 +136,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "c10348d26e3e3dc9e345d9044db35db9906617eb9c98aaf58a55d9681ffb85e3"
--- a/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json
+++ b/backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json
@@ -42,7 +42,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-d21e5be1ac26db926b7196316dae0c9fa82a865b95bb55de5370e4ee34889ba3.json
+++ b/backend/.sqlx/query-d21e5be1ac26db926b7196316dae0c9fa82a865b95bb55de5370e4ee34889ba3.json
@@ -1,16 +1,17 @@
 {
  "db_name": "PostgreSQL",
-  "query": "UPDATE workspace_settings SET error_handler = $1, error_handler_extra_args = $2 WHERE workspace_id = $3",
+  "query": "UPDATE workspace_settings SET error_handler = $1, error_handler_extra_args = $2, error_handler_muted_on_cancel = $3 WHERE workspace_id = $4",
  "describe": {
    "columns": [],
    "parameters": {
      "Left": [
        "Varchar",
        "Json",
+        "Bool",
        "Text"
      ]
    },
    "nullable": []
  },
-  "hash": "ea2ad5213bb6de8eab502fbb76149fa7964cd11e74e5ed1a30e857aad8ea5595"
+  "hash": "d21e5be1ac26db926b7196316dae0c9fa82a865b95bb55de5370e4ee34889ba3"
 }
--- a/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json
+++ b/backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json
@@ -37,7 +37,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
--- a/backend/.sqlx/query-dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb.json
+++ b/backend/.sqlx/query-dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -131,7 +136,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "dd74fa9468b5fe8c8ad657ded06076c11a78d0206af2af20685c691d4d9520bb"
--- a/backend/.sqlx/query-eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8.json
+++ b/backend/.sqlx/query-eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -131,7 +136,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "eced0a09ba547ce1dccb54a5419b22373603c9d01f77047b3553bde125bf71e8"
--- a/backend/.sqlx/query-f1f7a27738deae46de6acc10cbff701ff0c3872cc5b387e304bd9a058bb0c8a8.json
+++ b/backend/.sqlx/query-f1f7a27738deae46de6acc10cbff701ff0c3872cc5b387e304bd9a058bb0c8a8.json
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET  canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 RETURNING id",
+  "query": "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 RETURNING id",
  "describe": {
    "columns": [
      {
@@ -21,5 +21,5 @@
      false
    ]
  },
-  "hash": "960dce02871e9697d7aca2fcd5a75eefc3db7179e49fbf20c7e62cbcb235683e"
+  "hash": "f1f7a27738deae46de6acc10cbff701ff0c3872cc5b387e304bd9a058bb0c8a8"
 }
--- a/backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json
+++ b/backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select version from _sqlx_migrations order by version desc limit 1;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "version",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6"
+}
--- a/backend/.sqlx/query-fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480.json
+++ b/backend/.sqlx/query-fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480.json
@@ -102,6 +102,11 @@
        "ordinal": 19,
        "name": "on_recovery_extra_args",
        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
+        "name": "ws_error_handler_muted",
+        "type_info": "Bool"
      }
    ],
    "parameters": {
@@ -129,7 +134,8 @@
      true,
      true,
      true,
-      true
+      true,
+      false
    ]
  },
  "hash": "fdd3710a381dac33ef5ee5ec5564a4874e6d7807d43fa6ea7b8408cad9e97480"
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.191.0"
+version = "1.202.1"
 authors.workspace = true
 edition.workspace = true

@@ -21,7 +21,7 @@ members = [
 ]

 [workspace.package]
-version = "1.191.0"
+version = "1.202.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"

@@ -37,6 +37,8 @@ incremental = true
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
 benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
 flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
+loki = ["windmill-common/loki"]
+pg_embed = ["dep:pg-embed"]

 [dependencies]
 anyhow.workspace = true
@@ -63,6 +65,7 @@ uuid.workspace = true
 gethostname.workspace = true
 serde_json.workspace = true
 serde.workspace = true
+pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true} 

 [dev-dependencies]
 serde_json.workspace = true
--- a/backend/migrations/20231103143535_schedule_default_error_handler.down.sql
+++ b/backend/migrations/20231103143535_schedule_default_error_handler.down.sql
@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE schedule DROP COLUMN ws_error_handler_muted;
--- a/backend/migrations/20231103143535_schedule_default_error_handler.up.sql
+++ b/backend/migrations/20231103143535_schedule_default_error_handler.up.sql
@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN ws_error_handler_muted BOOLEAN NOT NULL DEFAULT false;
--- a/backend/migrations/20231105173726_add_worker_version.down.sql
+++ b/backend/migrations/20231105173726_add_worker_version.down.sql
@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE worker_ping DROP COLUMN wm_version;
--- a/backend/migrations/20231105173726_add_worker_version.up.sql
+++ b/backend/migrations/20231105173726_add_worker_version.up.sql
@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS wm_version VARCHAR(255) NOT NULL DEFAULT '';
--- a/backend/migrations/20231106131265_error_handler_skipped_on_cancel.down.sql
+++ b/backend/migrations/20231106131265_error_handler_skipped_on_cancel.down.sql
@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE workspace_settings DROP COLUMN error_handler_muted_on_cancel;
--- a/backend/migrations/20231106131265_error_handler_skipped_on_cancel.up.sql
+++ b/backend/migrations/20231106131265_error_handler_skipped_on_cancel.up.sql
@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS error_handler_muted_on_cancel BOOL NOT NULL DEFAULT false;
--- a/frontend/src/routes/(root)/(logged)/scripts/dev/+page.js
+++ b/frontend/src/routes/(root)/(logged)/scripts/dev/+page.js
--- a/backend/openapi-deref.yaml
+++ b/backend/openapi-deref.yaml
--- a/backend/parsers/windmill-parser-py-imports/src/lib.rs
+++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs
@@ -42,7 +42,9 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
    "dns" => "dnspython",
    "msoffcrypto" => "msoffcrypto-tool",
    "tabula" => "tabula-py",
-    "shapefile" => "pyshp"
+    "shapefile" => "pyshp",
+    "sklearn" => "scikit-learn",
+    "umap" => "umap-learn"
 };

 fn replace_import(x: String) -> String {
--- a/backend/src/ee.rs
+++ b/backend/src/ee.rs
@@ -2,7 +2,8 @@
 use windmill_common::error;

 pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
-    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
+    use windmill_api::ee::validate_license_key;
+    use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};

    let id = validate_license_key(license_key.clone()).await?;
    {
@@ -24,7 +25,7 @@ pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {

 #[cfg(feature = "enterprise")]
 pub async fn verify_license_key() -> error::Result<()> {
-    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_VALID};
    use windmill_common::error::to_anyhow;

    let expiry_nb = LICENSE_KEY
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -24,13 +24,14 @@ use windmill_api::HTTP_CLIENT;
 use windmill_common::{
    global_settings::{
        BASE_URL_SETTING, CUSTOM_TAGS_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS,
-        EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
+        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
    },
    stats::schedule_stats,
    utils::rd_string,
    worker::{reload_custom_tags_setting, WORKER_GROUP},
-    DB, METRICS_ADDR,
+    DB, METRICS_ADDR, METRICS_ENABLED,
 };
 use windmill_worker::{
    BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
@@ -40,9 +41,9 @@ use windmill_worker::{
 };

 use crate::monitor::{
-    initial_load, monitor_db, reload_base_url_setting, reload_extra_pip_index_url_setting,
-    reload_license_key, reload_npm_config_registry_setting, reload_retention_period_setting,
-    reload_server_config, reload_worker_config,
+    initial_load, load_keep_job_dir, monitor_db, monitor_pool, reload_base_url_setting,
+    reload_extra_pip_index_url_setting, reload_license_key, reload_npm_config_registry_setting,
+    reload_retention_period_setting, reload_server_config, reload_worker_config,
 };

 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
@@ -53,6 +54,9 @@ const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 mod ee;
 mod monitor;

+#[cfg(feature = "pg_embed")]
+mod pg_embed;
+
 #[derive(Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum Mode {
@@ -65,6 +69,10 @@ pub enum Mode {
 async fn main() -> anyhow::Result<()> {
    dotenv::dotenv().ok();

+    if std::env::var("RUST_LOG").is_err() {
+        std::env::set_var("RUST_LOG", "info")
+    }
+
    #[cfg(not(feature = "flamegraph"))]
    windmill_common::tracing_init::initialize_tracing();

@@ -124,7 +132,6 @@ async fn main() -> anyhow::Result<()> {
            "We STRONGLY recommend using at most 1 worker per container, use at your own risks"
        );
    }
-    let metrics_addr: Option<SocketAddr> = *METRICS_ADDR;

    let server_mode = !std::env::var("DISABLE_SERVER")
        .ok()
@@ -162,6 +169,14 @@ async fn main() -> anyhow::Result<()> {
        config
    });

+    #[cfg(feature = "pg_embed")]
+    let _pg = {
+        let (db_url, pg) = pg_embed::start().await.expect("pg embed");
+        tracing::info!("Use embedded pg: {db_url}");
+        std::env::set_var("DATABASE_URL", db_url);
+        pg
+    };
+
    tracing::info!("Connecting to database...");
    let db = windmill_common::connect_db(server_mode).await?;
    tracing::info!("Database connected");
@@ -183,9 +198,31 @@ async fn main() -> anyhow::Result<()> {
        None
    };

+    let last_mig_version =
+        sqlx::query_scalar!("select version from _sqlx_migrations order by version desc limit 1;")
+            .fetch_optional(&db)
+            .await
+            .ok()
+            .flatten();
+
+    tracing::info!(
+        "Last migration version: {last_mig_version:?}. Starting potential migration of the db if first connection on a new windmill version (can take a while depending on the migration) ...",
+    );
+
    // migration code to avoid break
    windmill_api::migrate_db(&db).await?;

+    let last_mig_version =
+        sqlx::query_scalar!("select version from _sqlx_migrations order by version desc limit 1;")
+            .fetch_optional(&db)
+            .await
+            .ok()
+            .flatten();
+
+    tracing::info!(
+        "Completed potential migration of the db. Last migration version: {last_mig_version:?}",
+    );
+
    let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
    let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());

@@ -225,6 +262,8 @@ Windmill Community Edition {GIT_VERSION}

        monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;

+        monitor_pool(&db).await;
+
        if std::env::var("BASE_INTERNAL_URL").is_ok() {
            tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
        }
@@ -241,7 +280,15 @@ Windmill Community Edition {GIT_VERSION}
            .expect("could not create initial server dir");

        let server_f = async {
-            windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe(), port_tx).await?;
+            windmill_api::run_server(
+                db.clone(),
+                rsmq2,
+                addr,
+                rx.resubscribe(),
+                port_tx,
+                server_mode,
+            )
+            .await?;
            Ok(()) as anyhow::Result<()>
        };

@@ -338,13 +385,29 @@ Windmill Community Edition {GIT_VERSION}
                                                NPM_CONFIG_REGISTRY_SETTING => {
                                                    reload_npm_config_registry_setting(&db).await
                                                },
+                                                KEEP_JOB_DIR_SETTING => {
+                                                    load_keep_job_dir(&db).await;
+                                                }
+                                                EXPOSE_METRICS_SETTING | EXPOSE_DEBUG_METRICS_SETTING => {
+                                                    if n.payload() != EXPOSE_DEBUG_METRICS_SETTING || worker_mode {
+                                                        tracing::info!("Metrics setting changed, restarting");
+                                                        // we wait a bit randomly to avoid having all serverss and workers shutdown at same time
+                                                        let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                        tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                        if let Err(e) = tx.send(()) {
+                                                            tracing::error!(error = %e, "Could not send killpill to server");
+                                                        }
+                                                    }
+                                                },
                                                REQUEST_SIZE_LIMIT_SETTING => {
-                                                    tracing::info!("Request limit size change detected, killing server expecting to be restarted");
-                                                    // we wait a bit randomly to avoid having all servers shutdown at same time
-                                                    let rd_delay = rand::thread_rng().gen_range(0..4);
-                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
-                                                    if let Err(e) = tx.send(()) {
-                                                        tracing::error!(error = %e, "Could not send killpill to server");
+                                                    if server_mode {
+                                                        tracing::info!("Request limit size change detected, killing server expecting to be restarted");
+                                                        // we wait a bit randomly to avoid having all servers shutdown at same time
+                                                        let rd_delay = rand::thread_rng().gen_range(0..4);
+                                                        tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                        if let Err(e) = tx.send(()) {
+                                                            tracing::error!(error = %e, "Could not send killpill to server");
+                                                        }
                                                    }
                                                },
                                                DISABLE_STATS_SETTING => {},
@@ -381,12 +444,13 @@ Windmill Community Edition {GIT_VERSION}
        };

        let metrics_f = async {
-            if let Some(_addr) = metrics_addr {
+            if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
                #[cfg(not(feature = "enterprise"))]
-                panic!("Metrics are only available in the Enterprise Edition");
+                tracing::error!("Metrics are only available in the EE, ignoring...");

                #[cfg(feature = "enterprise")]
-                windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
+                windmill_common::serve_metrics(*METRICS_ADDR, rx.resubscribe(), num_workers > 0)
+                    .await;
            }
            Ok(()) as anyhow::Result<()>
        };
@@ -520,7 +584,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
    for i in 1..(num_workers + 1) {
        let db1 = db.clone();
        let instance_name = instance_name.clone();
-        let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
+        let worker_name = format!("wk-{}-{}-{}", *WORKER_GROUP, &instance_name, rd_string(5));
        let ip = ip.clone();
        let rx = rx.resubscribe();
        let tx = tx.clone();
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -1,4 +1,11 @@
-use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc, time::Duration};
+use std::{
+    collections::HashMap,
+    fmt::Display,
+    ops::Mul,
+    str::FromStr,
+    sync::{atomic::Ordering, Arc},
+    time::Duration,
+};

 use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
@@ -14,7 +21,8 @@ use windmill_api::{
 use windmill_common::{
    error,
    global_settings::{
-        BASE_URL_SETTING, EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING,
+        BASE_URL_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
        NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
        RETENTION_PERIOD_SECS_SETTING,
    },
@@ -22,10 +30,10 @@ use windmill_common::{
    server::load_server_config,
    users::truncate_token,
    worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
-    BASE_URL, DB, METRICS_ENABLED,
+    BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, NPM_CONFIG_REGISTRY,
+    create_token_for_owner, handle_job_error, AuthedClient, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY,
    PIP_EXTRA_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };

@@ -33,7 +41,7 @@ use windmill_worker::{
 use crate::ee::verify_license_key;

 #[cfg(feature = "enterprise")]
-use windmill_api::LICENSE_KEY_VALID;
+use windmill_common::ee::LICENSE_KEY_VALID;

 use crate::ee::set_license_key;

@@ -76,72 +84,98 @@ pub async fn initial_load(
    worker_mode: bool,
    server_mode: bool,
 ) {
-    let reload_worker_config_f = async {
-        if worker_mode {
-            reload_worker_config(&db, tx, false).await;
-        }
-    };
-    let reload_custom_tags_f = async {
-        if server_mode {
-            if let Err(e) = reload_custom_tags_setting(db).await {
-                tracing::error!("Error reloading custom tags: {:?}", e)
-            }
-        }
-    };
+    if let Err(e) = load_metrics_enabled(db).await {
+        tracing::error!("Error loading expose metrics: {e}");
+    }

-    let reload_base_url_f = async {
-        if let Err(e) = reload_base_url_setting(db).await {
-            tracing::error!("Error reloading base url: {:?}", e)
-        }
-    };
+    if let Err(e) = load_metrics_debug_enabled(db).await {
+        tracing::error!("Error loading expose debug metrics: {e}");
+    }

-    let reload_server_config_f = async {
-        if server_mode {
-            reload_server_config(&db).await;
-        }
-    };
-    let reload_retention_period_f = async {
-        if server_mode {
-            reload_retention_period_setting(&db).await;
-        }
-    };
+    if worker_mode {
+        load_keep_job_dir(db).await;
+    }

-    let reload_request_size_f = async {
-        if server_mode {
-            reload_request_size(&db).await;
-        }
-    };
+    if worker_mode {
+        reload_worker_config(&db, tx, false).await;
+    }

-    let reload_license_key_f = async {
-        #[cfg(feature = "enterprise")]
-        if let Err(e) = reload_license_key(&db).await {
-            tracing::error!("Error reloading license key: {:?}", e)
+    if server_mode {
+        if let Err(e) = reload_custom_tags_setting(db).await {
+            tracing::error!("Error reloading custom tags: {:?}", e)
        }
-    };
+    }

-    let reload_extra_pip_index_url_f = async {
-        if worker_mode {
-            reload_extra_pip_index_url_setting(&db).await;
+    if let Err(e) = reload_base_url_setting(db).await {
+        tracing::error!("Error reloading base url: {:?}", e)
+    }
+
+    if server_mode {
+        reload_server_config(&db).await;
+    }
+
+    if server_mode {
+        reload_retention_period_setting(&db).await;
+    }
+    if server_mode {
+        reload_request_size(&db).await;
+    }
+
+    #[cfg(feature = "enterprise")]
+    if let Err(e) = reload_license_key(&db).await {
+        tracing::error!("Error reloading license key: {:?}", e)
+    }
+
+    if worker_mode {
+        reload_extra_pip_index_url_setting(&db).await;
+    }
+
+    if worker_mode {
+        reload_npm_config_registry_setting(&db).await;
+    }
+}
+
+pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
+    let metrics_enabled = sqlx::query_scalar!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        EXPOSE_METRICS_SETTING
+    )
+    .fetch_optional(db)
+    .await;
+    match metrics_enabled {
+        Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed),
+        _ => (),
+    };
+    Ok(())
+}
+
+pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
+    let metrics_enabled = sqlx::query_scalar!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        EXPOSE_DEBUG_METRICS_SETTING
+    )
+    .fetch_optional(db)
+    .await;
+    match metrics_enabled {
+        Ok(Some(serde_json::Value::Bool(t))) => METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed),
+        _ => (),
+    };
+    Ok(())
+}
+pub async fn load_keep_job_dir(db: &DB) {
+    let metrics_enabled = sqlx::query_scalar!(
+        "SELECT value FROM global_settings WHERE name = $1",
+        KEEP_JOB_DIR_SETTING
+    )
+    .fetch_optional(db)
+    .await;
+    match metrics_enabled {
+        Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed),
+        Err(e) => {
+            tracing::error!("Error loading keep job dir metrics: {e}");
        }
+        _ => (),
    };
-
-    let reload_npm_config_registry_f = async {
-        if worker_mode {
-            reload_npm_config_registry_setting(&db).await;
-        }
-    };
-
-    join!(
-        reload_worker_config_f,
-        reload_server_config_f,
-        reload_custom_tags_f,
-        reload_request_size_f,
-        reload_base_url_f,
-        reload_retention_period_f,
-        reload_license_key_f,
-        reload_extra_pip_index_url_f,
-        reload_npm_config_registry_f
-    );
 }

 pub async fn delete_expired_items(db: &DB) -> () {
@@ -380,6 +414,38 @@ pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
    Ok(())
 }

+pub async fn monitor_pool(db: &DB) {
+    if METRICS_ENABLED.load(Ordering::Relaxed) {
+        let db = db.clone();
+        tokio::spawn(async move {
+            let active_pool_connections: prometheus::IntGauge = prometheus::register_int_gauge!(
+                "pool_connections_active",
+                "Number of active postgresql connections in the pool"
+            )
+            .unwrap();
+
+            let idle_pool_connections: prometheus::IntGauge = prometheus::register_int_gauge!(
+                "pool_connections_idle",
+                "Number of idle postgresql connections in the pool"
+            )
+            .unwrap();
+
+            let max_pool_connections: prometheus::IntGauge = prometheus::register_int_gauge!(
+                "pool_connections_max",
+                "Number of max postgresql connections in the pool"
+            )
+            .unwrap();
+
+            max_pool_connections.set(db.options().get_max_connections() as i64);
+            loop {
+                active_pool_connections.set(db.size() as i64);
+                idle_pool_connections.set(db.num_idle() as i64);
+                tokio::time::sleep(Duration::from_secs(30)).await;
+            }
+        });
+    }
+}
+
 pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
    db: &Pool<Postgres>,
    base_internal_url: &str,
@@ -388,7 +454,7 @@ pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 ) {
    let zombie_jobs_f = async {
        if server_mode {
-            handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+            handle_zombie_jobs(db, base_internal_url, rsmq.clone(), "server").await;
        }
    };
    let expired_items_f = async {
@@ -413,7 +479,7 @@ pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
    };

    let expose_queue_metrics_f = async {
-        if *METRICS_ENABLED && server_mode {
+        if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) && server_mode {
            expose_queue_metrics(&db).await;
        }
    };
@@ -574,6 +640,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
    db: &Pool<Postgres>,
    base_internal_url: &str,
    rsmq: Option<R>,
+    worker_name: &str,
 ) {
    if *RESTART_ZOMBIE_JOBS {
        let restarted = sqlx::query!(
@@ -587,7 +654,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
            .ok()
            .unwrap_or_else(|| vec![]);

-        if *METRICS_ENABLED {
+        if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
            QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
        }
        for r in restarted {
@@ -613,7 +680,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
        .ok()
        .unwrap_or_else(|| vec![]);

-    if *METRICS_ENABLED {
+    if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
        QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
    }

@@ -638,6 +705,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
            base_internal_url: base_internal_url.to_string(),
            token,
            workspace: job.workspace_id.to_string(),
+            force_client: None,
        };

        let last_ping = job.last_ping.clone();
@@ -646,6 +714,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
            &client,
            &job,
            0,
+            None,
            error::Error::ExecutionErr(format!(
                "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
                last_ping
@@ -653,11 +722,11 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                    .unwrap_or_else(|| "no ping".to_string()),
                *ZOMBIE_JOB_TIMEOUT
            )),
-            None,
            true,
            same_worker_tx_never_used,
            "",
            rsmq.clone(),
+            worker_name,
        )
        .await;
    }
--- a/backend/src/pg_embed.rs
+++ b/backend/src/pg_embed.rs
@@ -0,0 +1,45 @@
+use pg_embed::pg_enums::PgAuthMethod;
+use pg_embed::pg_fetch::PgFetchSettings;
+use pg_embed::postgres::{PgEmbed, PgSettings};
+use std::path::PathBuf;
+use std::time::Duration;
+
+pub async fn start() -> anyhow::Result<(String, PgEmbed)> {
+    let pg_settings = PgSettings {
+        database_dir: PathBuf::from("/tmp/db"),
+        port: 6543,
+        user: "postgres".to_string(),
+        password: "password".to_string(),
+        auth_method: PgAuthMethod::Plain,
+        persistent: false,
+        timeout: Some(Duration::from_secs(15)),
+        migration_dir: None,
+    };
+
+    let fetch_settings = PgFetchSettings {
+        version: pg_embed::pg_fetch::PostgresVersion("15.3.0"),
+
+        ..Default::default()
+    };
+
+    tracing::info!(
+        "Fetch settings: {:?} {:?}",
+        fetch_settings.operating_system,
+        fetch_settings.architecture
+    );
+
+    let mut pg = PgEmbed::new(pg_settings, fetch_settings).await?;
+
+    pg.setup().await.expect("pg setup");
+
+    pg.start_db().await.expect("pg start db");
+
+    if !pg.database_exists("windmill").await.expect("db exists") {
+        pg.create_database("windmill")
+            .await
+            .expect("pg create database");
+    }
+
+    let uri = pg.full_db_uri("windmill");
+    Ok((uri, pg))
+}
--- a/backend/tests/worker.rs
+++ b/backend/tests/worker.rs
@@ -28,7 +28,7 @@ use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};

 use windmill_common::worker::{WORKER_CONFIG, PriorityTags};
 use windmill_common::{
-    flow_status::{FlowStatus, FlowStatusModule},
+    flow_status::{FlowStatus, FlowStatusModule, RestartedFrom},
    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
    jobs::{JobPayload, RawCode, JobKind},
    scripts::{ScriptLang, ScriptHash}
@@ -132,6 +132,7 @@ impl ApiServer {
            addr,
            rx,
            port_tx,
+            false,
        ));

        _port_rx.await.unwrap();
@@ -276,7 +277,7 @@ mod suspend_resume {
        let server = ApiServer::start(db.clone()).await;
        let port = server.addr.port();

-        let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None })
+        let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None, restarted_from: None })
            .arg("n", json!(1))
            .arg("port", json!(port))
            .push(&db)
@@ -357,7 +358,7 @@ mod suspend_resume {
        let server = ApiServer::start(db.clone()).await;
        let port = server.addr.port();

-        let result = RunJob::from(JobPayload::RawFlow { value: flow(), path: None })
+        let result = RunJob::from(JobPayload::RawFlow { value: flow(), path: None, restarted_from: None })
            .arg("n", json!(1))
            .arg("op", json!("cancel"))
            .arg("port", json!(port))
@@ -381,7 +382,7 @@ mod suspend_resume {
        let server = ApiServer::start(db.clone()).await;
        let port = server.addr.port();

-        let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None })
+        let flow = RunJob::from(JobPayload::RawFlow { value: flow(), path: None, restarted_from: None })
            .arg("n", json!(1))
            .arg("port", json!(port))
            .push(&db)
@@ -576,7 +577,7 @@ def main(last, port):
        .into_iter()
        .unzip::<_, _, Vec<_>, Vec<_>>();
        let server = Server::start(responses).await;
-        let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
+        let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None, restarted_from: None })
            .arg("items", json!(["unused", "unused", "unused"]))
            .arg("port", json!(server.addr.port()))
            .run_until_complete(&db, server.addr.port())
@@ -605,7 +606,7 @@ def main(last, port):
        .into_iter()
        .unzip::<_, _, Vec<_>, Vec<_>>();
        let server = Server::start(responses).await;
-        let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
+        let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None, restarted_from: None })
            .arg("items", json!(["unused", "unused", "unused"]))
            .arg("port", json!(server.addr.port()))
            .run_until_complete(&db, server.addr.port())
@@ -646,7 +647,7 @@ def main(last, port):
        .into_iter()
        .unzip::<_, _, Vec<_>, Vec<_>>();
        let server = Server::start(responses).await;
-        let job = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
+        let job = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None, restarted_from: None })
            .arg("items", json!(["unused", "unused", "unused"]))
            .arg("port", json!(server.addr.port()))
            .run_until_complete(&db, server.addr.port())
@@ -712,7 +713,7 @@ def main(error, port):
        .into_iter()
        .unzip::<_, _, Vec<_>, Vec<_>>();
        let server = Server::start(responses).await;
-        let cjob = RunJob::from(JobPayload::RawFlow { value, path: None })
+        let cjob = RunJob::from(JobPayload::RawFlow { value, path: None, restarted_from: None })
            .arg("port", json!(server.addr.port()))
            .run_until_complete(&db, server.addr.port())
            .await;
@@ -770,7 +771,7 @@ async fn test_iteration(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("items", json!([]))
        .run_until_complete(&db, server.addr.port())
        .await
@@ -779,7 +780,7 @@ async fn test_iteration(db: Pool<Postgres>) {
    assert_eq!(result, serde_json::json!([]));

    /* Don't actually test that this does 257 jobs or that will take forever. */
-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("items", json!((0..257).collect::<Vec<_>>()))
        .run_until_complete(&db, server.addr.port())
        .await
@@ -827,7 +828,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("items", json!([]))
        .run_until_complete(&db, server.addr.port())
        .await
@@ -836,7 +837,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
    assert_eq!(result, serde_json::json!([]));

    /* Don't actually test that this does 257 jobs or that will take forever. */
-    let job = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let job = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("items", json!((0..50).collect::<Vec<_>>()))
        .run_until_complete(&db, server.addr.port())
        .await;
@@ -1136,7 +1137,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
        }
    };

-    let job = JobPayload::RawFlow { value: flow, path: None };
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let port = server.addr.port();

    for i in 0..50 {
@@ -1176,7 +1177,7 @@ async fn test_identity(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .run_until_complete(&db, server.addr.port())
        .await
        .json_result()
@@ -1365,7 +1366,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
            ..Default::default()
        };

-    let job = JobPayload::RawFlow { value: flow, path: None };
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };

    let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
        .await
@@ -1421,7 +1422,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
        }))
        .unwrap();

-    let job = JobPayload::RawFlow { value: flow, path: None };
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
        .await
        .json_result()
@@ -1463,7 +1464,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
        ],
    }))
    .unwrap();
-    let job = JobPayload::RawFlow { value: flow, path: None };
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };

    let result = RunJob::from(job.clone())
        .arg("n", json!(123))
@@ -1521,7 +1522,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
        ],
    }))
    .unwrap();
-    let job = JobPayload::RawFlow { value: flow, path: None };
+    let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };

    let result = RunJob::from(job.clone())
        .arg("n", json!(123))
@@ -1586,7 +1587,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
        println!("python flow iteration: {}", i);
        let result = run_job_in_new_worker_until_complete(
            &db,
-            JobPayload::RawFlow { value: flow.clone(), path: None },
+            JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None },
            port,
        )
        .await
@@ -1621,7 +1622,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
        println!("python flow iteration: {}", i);
        let result = run_job_in_new_worker_until_complete(
            &db,
-            JobPayload::RawFlow { value: flow.clone(), path: None },
+            JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None },
            port,
        )
        .await
@@ -1657,7 +1658,8 @@ func main(derp string) (string, error) {
        language: ScriptLang::Go,
        concurrent_limit: None,
        concurrency_time_window_s: None,
-        cache_ttl: None
+        cache_ttl: None,
+        dedicated_worker: None
    }))
    .arg("derp", json!("world"))
    .run_until_complete(&db, port)
@@ -1687,7 +1689,8 @@ echo "hello $msg"
        language: ScriptLang::Bash,
        concurrent_limit: None,
        concurrency_time_window_s: None,
-        cache_ttl: None
+        cache_ttl: None,
+        dedicated_worker: None
    }))
    .arg("msg", json!("world"))
    .run_until_complete(&db, port)
@@ -1714,7 +1717,8 @@ def main():
        lock: None,
        concurrent_limit: None,
        concurrency_time_window_s: None,
-        cache_ttl: None
+        cache_ttl: None,
+        dedicated_worker: None
    });

    let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -1747,7 +1751,8 @@ def main():
        lock: None,
        concurrent_limit: None,
        concurrency_time_window_s: None,
-        cache_ttl: None
+        cache_ttl: None,
+        dedicated_worker: None
    });

    let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -1779,7 +1784,8 @@ def main():
        lock: None,
        concurrent_limit: None,
        concurrency_time_window_s: None,
-        cache_ttl: None
+        cache_ttl: None,
+        dedicated_worker: None
    });

    let result = run_job_in_new_worker_until_complete(&db, job, port)
@@ -1837,7 +1843,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -1876,7 +1882,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let job = run_job_in_new_worker_until_complete(&db, flow, port).await;

    assert_eq!(
@@ -1918,7 +1924,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -1973,7 +1979,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2041,7 +2047,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2077,7 +2083,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2115,7 +2121,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2152,7 +2158,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2199,7 +2205,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2236,7 +2242,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2300,7 +2306,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2357,7 +2363,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
    }))
    .unwrap();

-    let flow = JobPayload::RawFlow { value: flow, path: None };
+    let flow = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
    let result = run_job_in_new_worker_until_complete(&db, flow, port)
        .await
        .json_result()
@@ -2421,7 +2427,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
        }))
        .unwrap();

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("n", json!(0))
        .run_until_complete(&db, port)
        .await
@@ -2437,7 +2443,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
        .unwrap()
        .contains("[]"));

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("n", json!(1))
        .run_until_complete(&db, port)
        .await
@@ -2453,7 +2459,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
        .unwrap()
        .contains("[0]"));

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("n", json!(2))
        .run_until_complete(&db, port)
        .await
@@ -2469,7 +2475,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
        .unwrap()
        .contains("[0,1]"));

-    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
+    let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None })
        .arg("n", json!(3))
        .run_until_complete(&db, port)
        .await
@@ -2610,6 +2616,197 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
        });
 }

+#[sqlx::test(fixtures("base"))]
+
+async fn test_complex_flow_restart(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let flow: FlowValue = serde_json::from_value(json!({
+        "modules": [
+            {
+                "id": "a",
+                "value": {
+                    "type": "rawscript",
+                    "language": "go",
+                    "content": "package inner\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n)\nfunc main(max int) (interface{}, error) {\n\tresult := rand.Intn(max) + 1\n\tfmt.Printf(\"Number generated: '%d'\", result)\n\treturn result, nil\n}",
+                    "input_transforms": {
+                        "max": {
+                            "type": "static",
+                            "value": json!(20),
+                        },
+                    }
+                },
+                "summary": "Generate random number in [1, 20]"
+            },
+            {
+                "id": "b",
+                "value":
+                {
+                    "type": "branchall",
+                    "branches":
+                    [
+                        {
+                            "modules":
+                            [
+                                {
+                                    "id": "d",
+                                    "value":
+                                    {
+                                        "type": "branchone",
+                                        "default":
+                                        [
+                                            {
+                                                "id": "f",
+                                                "value":
+                                                {
+                                                    "type": "rawscript",
+                                                    "content": "package inner\nimport \"math/rand\"\nfunc main(x int) (interface{}, error) {\n\treturn rand.Intn(x) + 1, nil\n}",
+                                                    "language": "go",
+                                                    "input_transforms":
+                                                    {
+                                                        "x":
+                                                        {
+                                                            "expr": "results.a",
+                                                            "type": "javascript"
+                                                        }
+                                                    }
+                                                },
+                                                "summary": "Rand N in [1; x]"
+                                            }
+                                        ],
+                                        "branches":
+                                        [
+                                            {
+                                                "expr": "results.a < flow_input.max / 2",
+                                                "modules":
+                                                [
+                                                    {
+                                                        "id": "e",
+                                                        "value":
+                                                        {
+                                                            "type": "rawscript",
+                                                            "content": "package inner\nimport \"math/rand\"\nfunc main(x int) (interface{}, error) {\n\treturn rand.Intn(x * 2) + 1, nil\n}\n",
+                                                            "language": "go",
+                                                            "input_transforms":
+                                                            {
+                                                                "x":
+                                                                {
+                                                                    "expr": "results.a",
+                                                                    "type": "javascript"
+                                                                }
+                                                            }
+                                                        },
+                                                        "summary": "Rand N in [1; x*2]"
+                                                    }
+                                                ],
+                                                "summary": "N in first half"
+                                            }
+                                        ]
+                                    },
+                                    "summary": ""
+                                }
+                            ],
+                            "summary": "Process x",
+                            "parallel": true,
+                            "skip_failure": false
+                        },
+                        {
+                            "modules":
+                            [
+                                {
+                                    "id": "c",
+                                    "value":
+                                    {
+                                        "type": "rawscript",
+                                        "content": "package inner\nfunc main(x int) (interface{}, error) {\n\treturn x, nil\n}",
+                                        "language": "go",
+                                        "input_transforms":
+                                        {
+                                            "x":
+                                            {
+                                                "expr": "results.a",
+                                                "type": "javascript"
+                                            }
+                                        }
+                                    },
+                                    "summary": "Identity"
+                                }
+                            ],
+                            "summary": "Do nothing",
+                            "parallel": true,
+                            "skip_failure": false
+                        }
+                    ],
+                    "parallel": false
+                },
+                "summary": ""
+            },
+            {
+                "id": "g",
+                "value":
+                {
+                    "tag": "",
+                    "type": "rawscript",
+                    "content": "package inner\nimport \"fmt\"\nfunc main(x []int) (interface{}, error) {\n\tfmt.Printf(\"Results: %v\", x)\n\treturn x, nil\n}\n",
+                    "language": "go",
+                    "input_transforms":
+                    {
+                        "x":
+                        {
+                            "expr": "results.b",
+                            "type": "javascript"
+                        }
+                    }
+                },
+                "summary": "Print results - This will get the results from the prior step directly"
+            },
+            {
+                "id": "h",
+                "value":
+                {
+                    "tag": "",
+                    "type": "rawscript",
+                    "content": "package inner\nimport (\n\t\"fmt\"\n\t\"slices\"\n)\nfunc main(x []int) (interface{}, error) {\n\tresult := slices.Max(x)\n\tfmt.Printf(\"Result is %d\", result)\n\treturn result, nil\n}",
+                    "language": "go",
+                    "input_transforms":
+                    {
+                        "x":
+                        {
+                            "expr": "results.b",
+                            "type": "javascript"
+                        }
+                    }
+                },
+                "summary": "Choose max - this will get results.b querying get_result_by_id on the backend"
+            }
+        ],
+    }))
+    .unwrap();
+
+    let first_run_result = RunJob::from(JobPayload::RawFlow { 
+        value: flow.clone(), 
+        path: None, 
+        restarted_from: None 
+    }).run_until_complete(&db, port).await;
+
+    let restarted_flow_result = RunJob::from(JobPayload::RawFlow {
+        value: flow.clone(),
+        path: None,
+        restarted_from: Some(RestartedFrom {
+            flow_job_id: first_run_result.id,
+            step_id: "h".to_owned(),
+            branch_or_iteration_n: None,
+        }),
+    }).run_until_complete(&db, port).await;
+
+    assert_eq!(
+        first_run_result.json_result().unwrap(),
+        restarted_flow_result.json_result().unwrap()
+    );
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_rust_client(db: Pool<Postgres>) {
    initialize_tracing().await;
@@ -2662,6 +2859,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
        script_path: "f/system/failing_script".to_string(),
        timezone: "UTC".to_string(),
        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+        ws_error_handler_muted: None,
    };

    let _ = client.create_schedule("test-workspace", &schedule).await;
@@ -2721,6 +2919,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
                on_recovery_extra_args: None,
                timezone: "UTC".to_string(),
                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+                ws_error_handler_muted: None,
            },
        )
        .await
@@ -2796,6 +2995,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
        script_path: "f/system/failing_flow".to_string(),
        timezone: "UTC".to_string(),
        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+        ws_error_handler_muted: None,
    };

    let _ = client.create_schedule("test-workspace", &schedule).await;
@@ -2856,6 +3056,7 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
                on_recovery_extra_args: None,
                timezone: "UTC".to_string(),
                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+                ws_error_handler_muted: None,
            },
        )
        .await
@@ -2981,6 +3182,7 @@ async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: Strin
            concurrent_limit: None,
            concurrency_time_window_s: None,
            cache_ttl: None,
+            dedicated_worker: None
        })).push(&db2).await;


--- a/backend/windmill-api/openapi-deref.yaml
+++ b/backend/windmill-api/openapi-deref.yaml
@@ -1,6 +1,6 @@
 openapi: 3.0.3
 info:
-  version: 1.188.1
+  version: 1.201.0
  title: Windmill API
  contact:
    name: Windmill Team
@@ -875,6 +875,19 @@ paths:
            'text/plain:':
              schema:
                type: string
+  /settings/send_stats:
+    post:
+      summary: send stats
+      operationId: sendStats
+      tags:
+        - setting
+      responses:
+        '200':
+          description: status
+          content:
+            'text/plain:':
+              schema:
+                type: string
  /users/email:
    get:
      summary: get current user email (if logged in)
@@ -1487,6 +1500,8 @@ paths:
                  error_handler_extra_args:
                    type: object
                    additionalProperties: &ref_14 {}
+                  error_handler_muted_on_cancel:
+                    type: boolean
                required:
                  - code_completion_enabled
  /w/{workspace}/workspaces/get_deploy_to:
@@ -1763,6 +1778,8 @@ paths:
                error_handler_extra_args:
                  type: object
                  additionalProperties: *ref_14
+                error_handler_muted_on_cancel:
+                  type: boolean
      responses:
        '200':
          description: status
@@ -2329,7 +2346,7 @@ paths:
            application/json:
              schema:
                type: object
-                properties: &ref_151
+                properties: &ref_152
                  access_token:
                    type: string
                  expires_in:
@@ -2340,7 +2357,7 @@ paths:
                    type: array
                    items:
                      type: string
-                required: &ref_152
+                required: &ref_153
                  - access_token
  /w/{workspace}/oauth/create_account:
    post:
@@ -3048,13 +3065,17 @@ paths:
                type: array
                items:
                  type: string
-  /embeddings/query_resource_types:
+  /w/{workspace}/embeddings/query_resource_types:
    get:
      summary: query resource types by similarity
      operationId: queryResourceTypes
      tags:
        - resource
      parameters:
+        - name: workspace
+          in: path
+          required: true
+          schema: *ref_0
        - name: text
          description: query text
          in: query
@@ -3189,32 +3210,32 @@ paths:
                                id:
                                  type: string
                                value:
-                                  oneOf: &ref_173
+                                  oneOf: &ref_174
                                    - type: object
-                                      properties: &ref_157
+                                      properties: &ref_158
                                        input_transforms:
                                          type: object
                                          additionalProperties:
                                            oneOf: &ref_23
                                              - type: object
-                                                properties: &ref_153
+                                                properties: &ref_154
                                                  value: {}
                                                  type:
                                                    type: string
                                                    enum:
                                                      - javascript
-                                                required: &ref_154
+                                                required: &ref_155
                                                  - expr
                                                  - type
                                              - type: object
-                                                properties: &ref_155
+                                                properties: &ref_156
                                                  expr:
                                                    type: string
                                                  type:
                                                    type: string
                                                    enum:
                                                      - javascript
-                                                required: &ref_156
+                                                required: &ref_157
                                                  - expr
                                                  - type
                                            discriminator: &ref_24
@@ -3253,13 +3274,13 @@ paths:
                                          type: number
                                        concurrency_time_window_s:
                                          type: number
-                                      required: &ref_158
+                                      required: &ref_159
                                        - type
                                        - content
                                        - language
                                        - input_transforms
                                    - type: object
-                                      properties: &ref_159
+                                      properties: &ref_160
                                        input_transforms:
                                          type: object
                                          additionalProperties:
@@ -3273,12 +3294,12 @@ paths:
                                          type: string
                                          enum:
                                            - script
-                                      required: &ref_160
+                                      required: &ref_161
                                        - type
                                        - path
                                        - input_transforms
                                    - type: object
-                                      properties: &ref_161
+                                      properties: &ref_162
                                        input_transforms:
                                          type: object
                                          additionalProperties:
@@ -3290,12 +3311,12 @@ paths:
                                          type: string
                                          enum:
                                            - flow
-                                      required: &ref_162
+                                      required: &ref_163
                                        - type
                                        - path
                                        - input_transforms
                                    - type: object
-                                      properties: &ref_163
+                                      properties: &ref_164
                                        modules:
                                          type: array
                                          items:
@@ -3317,13 +3338,13 @@ paths:
                                          type: boolean
                                        parallelism:
                                          type: integer
-                                      required: &ref_164
+                                      required: &ref_165
                                        - modules
                                        - iterator
                                        - skip_failures
                                        - type
                                    - type: object
-                                      properties: &ref_165
+                                      properties: &ref_166
                                        branches:
                                          type: array
                                          items:
@@ -3354,12 +3375,12 @@ paths:
                                          type: string
                                          enum:
                                            - branchone
-                                      required: &ref_166
+                                      required: &ref_167
                                        - branches
                                        - default
                                        - type
                                    - type: object
-                                      properties: &ref_167
+                                      properties: &ref_168
                                        branches:
                                          type: array
                                          items:
@@ -3384,28 +3405,28 @@ paths:
                                            - branchall
                                        parallel:
                                          type: boolean
-                                      required: &ref_168
+                                      required: &ref_169
                                        - branches
                                        - type
                                    - type: object
-                                      properties: &ref_169
+                                      properties: &ref_170
                                        type:
                                          type: string
                                          enum:
                                            - identity
                                        flow:
                                          type: boolean
-                                      required: &ref_170
+                                      required: &ref_171
                                        - type
                                    - type: object
-                                      properties: &ref_171
+                                      properties: &ref_172
                                        type:
                                          type: string
                                          enum:
                                            - graphql
-                                      required: &ref_172
+                                      required: &ref_173
                                        - type
-                                  discriminator: &ref_174
+                                  discriminator: &ref_175
                                    propertyName: type
                                    mapping:
                                      rawscript: '#/components/schemas/RawScript'
@@ -3461,7 +3482,7 @@ paths:
                                  type: number
                                retry:
                                  type: object
-                                  properties: &ref_175
+                                  properties: &ref_176
                                    constant:
                                      type: object
                                      properties:
@@ -3981,13 +4002,9 @@ paths:
                      items:
                        type: string
                    concurrent_limit:
-                      type: array
-                      items:
-                        type: integer
+                      type: integer
                    concurrency_time_window_s:
-                      type: array
-                      items:
-                        type: integer
+                      type: integer
                    cache_ttl:
                      type: number
                    dedicated_worker:
@@ -4169,13 +4186,9 @@ paths:
                  items:
                    type: string
                concurrent_limit:
-                  type: array
-                  items:
-                    type: integer
+                  type: integer
                concurrency_time_window_s:
-                  type: array
-                  items:
-                    type: integer
+                  type: integer
                cache_ttl:
                  type: number
                dedicated_worker:
@@ -5411,7 +5424,7 @@ paths:
                type: array
                items:
                  type: object
-                  properties: &ref_148
+                  properties: &ref_149
                    workspace_id:
                      type: string
                    path:
@@ -5429,7 +5442,7 @@ paths:
                    edited_at:
                      type: string
                      format: date-time
-                  required: &ref_149
+                  required: &ref_150
                    - workspace_id
                    - path
                    - summary
@@ -5565,7 +5578,7 @@ paths:
                type: array
                items:
                  type: object
-                  properties: &ref_146
+                  properties: &ref_147
                    id:
                      type: integer
                    workspace_id:
@@ -5591,7 +5604,7 @@ paths:
                        - viewer
                        - publisher
                        - anonymous
-                  required: &ref_147
+                  required: &ref_148
                    - id
                    - workspace_id
                    - path
@@ -5764,7 +5777,7 @@ paths:
          content:
            application/json:
              schema:
-                allOf: &ref_150
+                allOf: &ref_151
                  - type: object
                    properties: *ref_45
                    required: *ref_46
@@ -6142,6 +6155,92 @@ paths:
              schema:
                type: string
                format: uuid
+  /w/{workspace}/jobs/restart/f/{id}/from/{step_id}/{branch_or_iteration_n}:
+    post:
+      summary: restart a completed flow at a given step
+      operationId: restartFlowAtStep
+      tags:
+        - job
+      parameters:
+        - name: workspace
+          in: path
+          required: true
+          schema: *ref_0
+        - name: id
+          in: path
+          required: true
+          schema: &ref_69
+            type: string
+            format: uuid
+        - name: step_id
+          description: step id to restart the flow from
+          required: true
+          in: path
+          schema:
+            type: string
+        - name: branch_or_iteration_n
+          description: >-
+            for branchall or loop, the iteration at which the flow should
+            restart
+          required: true
+          in: path
+          schema:
+            type: integer
+        - name: scheduled_for
+          description: when to schedule this job (leave empty for immediate run)
+          in: query
+          schema:
+            type: string
+            format: date-time
+        - name: scheduled_in_secs
+          description: schedule the script to execute in the number of seconds starting now
+          in: query
+          schema:
+            type: integer
+        - name: parent_job
+          description: >-
+            The parent job that is at the origin and responsible for the
+            execution of this script if any
+          in: query
+          schema: *ref_34
+        - name: job_id
+          description: >-
+            The job id to assign to the created job. if missing, job is chosen
+            randomly using the ULID scheme. If a job id already exists in the
+            queue or as a completed job, the request to create one will fail
+            (Bad Request)
+          in: query
+          schema: *ref_35
+        - name: include_header
+          description: >
+            List of headers's keys (separated with ',') whove value are added to
+            the args
+
+            Header's key lowercased and '-'' replaced to '_' such that
+            'Content-Type' becomes the 'content_type' arg key
+          in: query
+          schema: *ref_36
+        - name: invisible_to_owner
+          description: make the run invisible to the the flow owner (default false)
+          in: query
+          schema:
+            type: boolean
+      requestBody:
+        description: flow args
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              additionalProperties: *ref_14
+      responses:
+        '201':
+          description: job created
+          content:
+            text/plain:
+              schema:
+                type: string
+                format: uuid
  /w/{workspace}/jobs/run/h/{hash}:
    post:
      summary: run script by hash
@@ -6282,6 +6381,8 @@ paths:
                    - code
                    - identity
                    - http
+                dedicated_worker:
+                  type: boolean
              required: &ref_122
                - args
      responses:
@@ -6344,6 +6445,16 @@ paths:
                  additionalProperties: *ref_14
                tag:
                  type: string
+                restarted_from:
+                  type: object
+                  properties: &ref_146
+                    flow_job_id:
+                      type: string
+                      format: uuid
+                    step_id:
+                      type: string
+                    branch_or_iteration_n:
+                      type: integer
              required: &ref_145
                - value
                - content
@@ -7049,7 +7160,7 @@ paths:
              schema:
                type: array
                items:
-                  allOf: &ref_69
+                  allOf: &ref_70
                    - oneOf:
                        - type: object
                          properties: *ref_65
@@ -7064,7 +7175,7 @@ paths:
                          enum:
                            - CompletedJob
                            - QueuedJob
-                  discriminator: &ref_70
+                  discriminator: &ref_71
                    propertyName: type
  /jobs/db_clock:
    get:
@@ -7093,17 +7204,15 @@ paths:
        - name: id
          in: path
          required: true
-          schema: &ref_71
-            type: string
-            format: uuid
+          schema: *ref_69
      responses:
        '200':
          description: job details
          content:
            application/json:
              schema:
-                allOf: *ref_69
-                discriminator: *ref_70
+                allOf: *ref_70
+                discriminator: *ref_71
  /w/{workspace}/jobs_u/get_logs/{id}:
    get:
      summary: get job logs
@@ -7118,7 +7227,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      responses:
        '200':
          description: job details
@@ -7140,7 +7249,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: running
          in: query
          schema:
@@ -7179,7 +7288,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      responses:
        '200':
          description: job details
@@ -7203,7 +7312,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      responses:
        '200':
          description: result
@@ -7224,7 +7333,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: get_started
          in: query
          schema: &ref_98
@@ -7259,7 +7368,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      responses:
        '200':
          description: job details
@@ -7283,7 +7392,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      requestBody:
        description: reason
        required: true
@@ -7315,7 +7424,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      requestBody:
        description: reason
        required: true
@@ -7347,7 +7456,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7378,7 +7487,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7420,7 +7529,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: payload
          description: >
            The base64 encoded payload that has been encoded as a JSON. e.g how
@@ -7463,7 +7572,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7505,7 +7614,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
      requestBody:
        required: true
        content:
@@ -7533,7 +7642,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7568,7 +7677,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7610,7 +7719,7 @@ paths:
        - name: id
          in: path
          required: true
-          schema: *ref_71
+          schema: *ref_69
        - name: resume_id
          in: path
          required: true
@@ -7634,8 +7743,8 @@ paths:
                type: object
                properties:
                  job:
-                    allOf: *ref_69
-                    discriminator: *ref_70
+                    allOf: *ref_70
+                    discriminator: *ref_71
                  approvers:
                    type: array
                    items:
@@ -7732,6 +7841,8 @@ paths:
                on_recovery_extra_args:
                  type: object
                  additionalProperties: *ref_14
+                ws_error_handler_muted:
+                  type: boolean
              required: &ref_133
                - path
                - schedule
@@ -7792,6 +7903,8 @@ paths:
                on_recovery_extra_args:
                  type: object
                  additionalProperties: *ref_14
+                ws_error_handler_muted:
+                  type: boolean
              required: &ref_135
                - schedule
                - timezone
@@ -7928,6 +8041,8 @@ paths:
                  on_recovery_extra_args:
                    type: object
                    additionalProperties: *ref_14
+                  ws_error_handler_muted:
+                    type: boolean
                required: &ref_74
                  - path
                  - edited_by
@@ -8048,6 +8163,49 @@ paths:
                              - id
                              - success
                              - duration_ms
+  /w/{workspace}/schedules/setdefaulthandler:
+    post:
+      summary: Set default error or recoevery handler
+      operationId: setDefaultErrorOrRecoveryHandler
+      tags:
+        - schedule
+      parameters:
+        - name: workspace
+          in: path
+          required: true
+          schema: *ref_0
+      requestBody:
+        description: Handler description
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                handler_type:
+                  type: string
+                  enum:
+                    - error
+                    - recovery
+                override_existing:
+                  type: boolean
+                path:
+                  type: string
+                extra_args:
+                  type: object
+                number_of_occurence:
+                  type: integer
+                number_of_occurence_exact:
+                  type: boolean
+                workspace_handler_muted:
+                  type: boolean
+              required:
+                - handler_type
+                - override_existing
+                - path
+      responses:
+        '201':
+          description: default error handler set
  /groups/list:
    get:
      summary: list instance groups
@@ -8672,6 +8830,8 @@ paths:
                        type: string
                    worker_group:
                      type: string
+                    wm_version:
+                      type: string
                  required: &ref_137
                    - worker
                    - worker_instance
@@ -8680,6 +8840,7 @@ paths:
                    - ip
                    - jobs_executed
                    - worker_group
+                    - wm_version
  /configs/list_worker_groups:
    get:
      summary: list worker groups
@@ -9249,7 +9410,7 @@ components:
      name: id
      in: path
      required: true
-      schema: *ref_71
+      schema: *ref_69
    Path:
      name: path
      in: path
@@ -9492,8 +9653,8 @@ components:
      properties: *ref_65
      required: *ref_66
    Job:
-      allOf: *ref_69
-      discriminator: *ref_70
+      allOf: *ref_70
+      discriminator: *ref_71
    User:
      type: object
      properties: *ref_8
@@ -9747,23 +9908,26 @@ components:
      type: object
      properties: *ref_144
      required: *ref_145
+    RestartedFrom:
+      type: object
+      properties: *ref_146
    Policy:
      type: object
      properties: *ref_44
    ListableApp:
      type: object
-      properties: *ref_146
-      required: *ref_147
+      properties: *ref_147
+      required: *ref_148
    ListableRawApp:
      type: object
-      properties: *ref_148
-      required: *ref_149
+      properties: *ref_149
+      required: *ref_150
    AppWithLastVersion:
      type: object
      properties: *ref_45
      required: *ref_46
    AppWithLastVersionWDraft:
-      allOf: *ref_150
+      allOf: *ref_151
    SlackToken:
      type: object
      properties:
@@ -9785,64 +9949,64 @@ components:
        - bot
    TokenResponse:
      type: object
-      properties: *ref_151
-      required: *ref_152
+      properties: *ref_152
+      required: *ref_153
    HubScriptKind:
      name: kind
      schema: *ref_28
    StaticTransform:
      type: object
-      properties: *ref_153
-      required: *ref_154
+      properties: *ref_154
+      required: *ref_155
    JavascriptTransform:
      type: object
-      properties: *ref_155
-      required: *ref_156
+      properties: *ref_156
+      required: *ref_157
    InputTransform:
      oneOf: *ref_23
      discriminator: *ref_24
    RawScript:
      type: object
-      properties: *ref_157
-      required: *ref_158
+      properties: *ref_158
+      required: *ref_159
    PathScript:
      type: object
-      properties: *ref_159
-      required: *ref_160
+      properties: *ref_160
+      required: *ref_161
    PathFlow:
      type: object
-      properties: *ref_161
-      required: *ref_162
+      properties: *ref_162
+      required: *ref_163
    FlowModule:
      type: object
      properties: *ref_25
      required: *ref_26
    ForloopFlow:
      type: object
-      properties: *ref_163
-      required: *ref_164
+      properties: *ref_164
+      required: *ref_165
    BranchOne:
      type: object
-      properties: *ref_165
-      required: *ref_166
+      properties: *ref_166
+      required: *ref_167
    BranchAll:
      type: object
-      properties: *ref_167
-      required: *ref_168
+      properties: *ref_168
+      required: *ref_169
    Identity:
      type: object
-      properties: *ref_169
-      required: *ref_170
+      properties: *ref_170
+      required: *ref_171
    Graphql:
      type: object
-      properties: *ref_171
-      required: *ref_172
+      properties: *ref_172
+      required: *ref_173
    FlowModuleValue:
-      oneOf: *ref_173
-      discriminator: *ref_174
+      oneOf: *ref_174
+      discriminator: *ref_175
    Retry:
      type: object
-      properties: *ref_175
+      properties: *ref_176
    FlowValue:
      type: object
      properties: *ref_47
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.3"

 info:
-  version: 1.191.0
+  version: 1.202.1
  title: Windmill API

  contact:
@@ -650,6 +650,20 @@ paths:
              schema:
                type: string

+  /settings/send_stats:
+    post:
+      summary: send stats
+      operationId: sendStats
+      tags:
+        - setting
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
  /users/email:
    get:
      summary: get current user email (if logged in)
@@ -1160,6 +1174,8 @@ paths:
                    type: string
                  error_handler_extra_args:
                    $ref: "#/components/schemas/ScriptArgs"
+                  error_handler_muted_on_cancel:
+                    type: boolean
                required:
                  - code_completion_enabled

@@ -1423,6 +1439,8 @@ paths:
                  type: string
                error_handler_extra_args:
                  $ref: "#/components/schemas/ScriptArgs"
+                error_handler_muted_on_cancel:
+                  type: boolean

      responses:
        "200":
@@ -2363,13 +2381,14 @@ paths:
                items:
                  type: string

-  /embeddings/query_resource_types:
+  /w/{workspace}/embeddings/query_resource_types:
    get:
      summary: query resource types by similarity
      operationId: queryResourceTypes
      tags:
        - resource
      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
        - name: text
          description: query text
          in: query
@@ -4229,6 +4248,66 @@ paths:
                type: string
                format: uuid

+  /w/{workspace}/jobs/restart/f/{id}/from/{step_id}/{branch_or_iteration_n}:
+    post:
+      summary: restart a completed flow at a given step
+      operationId: restartFlowAtStep
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+        - name: step_id
+          description: step id to restart the flow from
+          required: true
+          in: path
+          schema:
+            type: string
+        - name: branch_or_iteration_n
+          description:
+            for branchall or loop, the iteration at which the flow should
+            restart
+          required: true
+          in: path
+          schema:
+            type: integer
+        - name: scheduled_for
+          description: when to schedule this job (leave empty for immediate run)
+          in: query
+          schema:
+            type: string
+            format: date-time
+        - name: scheduled_in_secs
+          description: schedule the script to execute in the number of seconds starting now
+          in: query
+          schema:
+            type: integer
+        - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
+        - $ref: "#/components/parameters/IncludeHeader"
+        - name: invisible_to_owner
+          description: make the run invisible to the the flow owner (default false)
+          in: query
+          schema:
+            type: boolean
+
+      requestBody:
+        description: flow args
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ScriptArgs"
+
+      responses:
+        "201":
+          description: job created
+          content:
+            text/plain:
+              schema:
+                type: string
+                format: uuid
+
  /w/{workspace}/jobs/run/h/{hash}:
    post:
      summary: run script by hash
@@ -5219,6 +5298,45 @@ paths:
                items:
                  $ref: "#/components/schemas/ScheduleWJobs"

+  /w/{workspace}/schedules/setdefaulthandler:
+    post:
+      summary: Set default error or recoevery handler
+      operationId: setDefaultErrorOrRecoveryHandler
+      tags:
+        - schedule
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: Handler description
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                handler_type:
+                  type: string
+                  enum: ["error", "recovery"]
+                override_existing:
+                  type: boolean
+                path:
+                  type: string
+                extra_args:
+                  type: object
+                number_of_occurence:
+                  type: integer
+                number_of_occurence_exact:
+                  type: boolean
+                workspace_handler_muted:
+                  type: boolean
+              required:
+                - handler_type
+                - override_existing
+                - path
+      responses:
+        "201":
+          description: default error handler set
+
  /groups/list:
    get:
      summary: list instance groups
@@ -7370,7 +7488,8 @@ components:
        kind:
          type: string
          enum: [code, identity, http]
-
+        dedicated_worker:
+          type: boolean
      required:
        - args

@@ -7521,6 +7640,8 @@ components:
          type: number
        on_recovery_extra_args:
          $ref: "#/components/schemas/ScriptArgs"
+        ws_error_handler_muted:
+          type: boolean
      required:
        - path
        - edited_by
@@ -7586,6 +7707,8 @@ components:
          type: number
        on_recovery_extra_args:
          $ref: "#/components/schemas/ScriptArgs"
+        ws_error_handler_muted:
+          type: boolean
      required:
        - path
        - schedule
@@ -7618,6 +7741,8 @@ components:
          type: number
        on_recovery_extra_args:
          $ref: "#/components/schemas/ScriptArgs"
+        ws_error_handler_muted:
+          type: boolean
      required:
        - schedule
        - timezone
@@ -7695,6 +7820,8 @@ components:
            type: string
        worker_group:
          type: string
+        wm_version:
+          type: string
      required:
        - worker
        - worker_instance
@@ -7703,7 +7830,7 @@ components:
        - ip
        - jobs_executed
        - worker_group
-
+        - wm_version
    UserWorkspaceList:
      type: object
      properties:
@@ -7865,12 +7992,25 @@ components:
          $ref: "#/components/schemas/ScriptArgs"
        tag:
          type: string
+        restarted_from:
+          $ref: "#/components/schemas/RestartedFrom"

      required:
        - value
        - content
        - args

+    RestartedFrom:
+      type: object
+      properties:
+        flow_job_id:
+          type: string
+          format: uuid
+        step_id:
+          type: string
+        branch_or_iteration_n:
+          type: integer
+
    Policy:
      type: object
      properties:
--- a/backend/windmill-api/src/embeddings.rs
+++ b/backend/windmill-api/src/embeddings.rs
@@ -1,7 +1,11 @@
 use std::{collections::HashMap, path::PathBuf, sync::Arc};

 use anyhow::{self, Error, Result};
-use axum::{extract::Query, routing::get, Extension, Json, Router};
+use axum::{
+    extract::{Path, Query},
+    routing::get,
+    Extension, Json, Router,
+};
 use candle_core::{Device, Tensor};
 use candle_nn::VarBuilder;
 use candle_transformers::models::bert::{BertModel, Config, DTYPE};
@@ -70,13 +74,14 @@ pub struct ResourceTypeResult {
 }
 async fn query_resource_types(
    Query(query): Query<ResourceTypesQuery>,
+    Path(w_id): Path<String>,
    Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
 ) -> JsonResult<Vec<ResourceTypeResult>> {
    let embeddings_db = embeddings_db.read().await;

    if let Some(embeddings_db) = embeddings_db.as_ref() {
        let results = embeddings_db
-            .query_resource_types(&query.text, query.limit)
+            .query_resource_types(w_id, &query.text, query.limit)
            .await?;

        Ok(Json(results))
@@ -157,8 +162,6 @@ impl ModelInstance {
            Tokenizer::from_file(tokenizer_filename)
                .map_err(Error::msg)?
                .with_padding(None)
-                .with_truncation(None)
-                .map_err(Error::msg)?
                .to_owned(),
        );

@@ -264,13 +267,10 @@ impl EmbeddingsDb {
        .await?;
        let hub_resource_types = response.json::<Vec<HubResourceType>>().await?;

-        let resource_types: Vec<ResourceType> = sqlx::query_as!(
-            ResourceType,
-            "SELECT * from resource_type ORDER \
-             BY name",
-        )
-        .fetch_all(pg_db)
-        .await?;
+        let resource_types: Vec<ResourceType> =
+            sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",)
+                .fetch_all(pg_db)
+                .await?;

        for rt in resource_types {
            let mut hm = HashMap::new();
@@ -278,6 +278,7 @@ impl EmbeddingsDb {
            if let Some(schema) = rt.schema.clone() {
                hm.insert("schema".to_string(), serde_json::to_string(&schema)?);
            }
+            hm.insert("workspace".to_string(), rt.workspace_id.clone());
            let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name);

            let vector = if let Some(hub_rt) = hub_rt {
@@ -389,6 +390,7 @@ impl EmbeddingsDb {

    pub async fn query_resource_types(
        &self,
+        workspace: String,
        query: &str,
        limit: Option<i64>,
    ) -> Result<Vec<ResourceTypeResult>> {
@@ -402,10 +404,23 @@ impl EmbeddingsDb {
        }

        let collection = collection.ok_or(Error::msg("no collection found"))?;
+
+        let filter = |embedding: &Embedding| {
+            if let Some(metadata) = embedding.metadata.as_ref() {
+                match metadata.get("workspace").map(|x| x.as_str()) {
+                    Some("admins") => true,
+                    Some(rt_workspace) => &workspace == rt_workspace,
+                    _ => false,
+                }
+            } else {
+                false
+            }
+        };
+
        let results = collection.get_similarity(
            &query_embedding,
            limit.unwrap_or(10) as usize,
-            None,
+            Some(&filter),
            Some(0.75),
        );

@@ -439,7 +454,7 @@ fn normalize_l2(v: &Tensor) -> Result<Tensor> {
    Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?)
 }

-pub fn global_service(db: &Pool<Postgres>) -> Router {
+pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb>>> {
    let embeddings_db: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));

    let disable_embedding = std::env::var("DISABLE_EMBEDDING")
@@ -478,8 +493,25 @@ pub fn global_service(db: &Pool<Postgres>) -> Router {
        });
    }

-    Router::new()
-        .route("/query_hub_scripts", get(query_hub_scripts))
-        .route("/query_resource_types", get(query_resource_types))
-        .layer(Extension(embeddings_db))
+    embeddings_db
+}
+
+pub fn workspaced_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
+    if let Some(embeddings_db) = embeddings_db {
+        Router::new()
+            .route("/query_resource_types", get(query_resource_types))
+            .layer(Extension(embeddings_db))
+    } else {
+        Router::new()
+    }
+}
+
+pub fn global_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
+    if let Some(embeddings_db) = embeddings_db {
+        Router::new()
+            .route("/query_hub_scripts", get(query_hub_scripts))
+            .layer(Extension(embeddings_db))
+    } else {
+        Router::new()
+    }
 }
--- a/backend/windmill-api/src/jobs.rs
+++ b/backend/windmill-api/src/jobs.rs
@@ -8,6 +8,7 @@

 use serde_json::value::RawValue;
 use std::collections::HashMap;
+use windmill_common::flow_status::RestartedFrom;

 use crate::db::ApiAuthed;

@@ -41,7 +42,7 @@ use windmill_common::{
    error::{self, to_anyhow, Error},
    flow_status::{Approval, FlowStatus, FlowStatusModule},
    flows::FlowValue,
-    jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
+    jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode},
    oauth2::HmacSha256,
    scripts::{ScriptHash, ScriptLang},
    users::username_to_permissioned_as,
@@ -63,6 +64,14 @@ pub fn workspaced_service() -> Router {
                .head(|| async { "" })
                .layer(cors.clone()),
        )
+        .route(
+            "/restart/f/:job_id/from/:step_id",
+            post(restart_flow).head(|| async { "" }).layer(cors.clone()),
+        )
+        .route(
+            "/restart/f/:job_id/from/:step_id/:branch_of_iteration_n",
+            post(restart_flow).head(|| async { "" }).layer(cors.clone()),
+        )
        .route(
            "/run/p/*script_path",
            post(run_job_by_path)
@@ -398,64 +407,6 @@ async fn get_job_logs(
    Ok(text)
 }

-#[derive(Debug, sqlx::FromRow, Serialize)]
-pub struct CompletedJob {
-    pub workspace_id: String,
-    pub id: Uuid,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub parent_job: Option<Uuid>,
-    pub created_by: String,
-    pub created_at: chrono::DateTime<chrono::Utc>,
-    pub started_at: chrono::DateTime<chrono::Utc>,
-    pub duration_ms: i64,
-    pub success: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_hash: Option<ScriptHash>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_path: Option<String>,
-    pub args: Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub result: Option<sqlx::types::Json<Box<RawValue>>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub logs: Option<String>,
-    pub deleted: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_code: Option<String>,
-    pub canceled: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_by: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_reason: Option<String>,
-    pub job_kind: JobKind,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub schedule_path: Option<String>,
-    pub permissioned_as: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub flow_status: Option<sqlx::types::Json<Box<RawValue>>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_flow: Option<sqlx::types::Json<Box<RawValue>>>,
-    pub is_flow_step: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub language: Option<ScriptLang>,
-    pub is_skipped: bool,
-    pub email: String,
-    pub visible_to_owner: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mem_peak: Option<i32>,
-    pub tag: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub priority: Option<i16>,
-}
-
-impl CompletedJob {
-    pub fn json_result(&self) -> Option<serde_json::Value> {
-        self.result
-            .as_ref()
-            .map(|r| serde_json::from_str(r.get()).ok())
-            .flatten()
-    }
-}
-
 #[derive(Debug, sqlx::FromRow, Serialize)]
 pub struct ListableCompletedJob {
    pub r#type: String,
@@ -501,12 +452,6 @@ pub struct ListableCompletedJob {
    pub priority: Option<i16>,
 }

-impl<'a> IntoResponse for CompletedJob {
-    fn into_response(self) -> Response {
-        Json(self).into_response()
-    }
-}
-
 #[derive(Deserialize, Clone)]
 pub struct RunJobQuery {
    scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
@@ -1523,6 +1468,7 @@ struct Preview {
    args: Option<Box<JsonRawValue>>,
    language: Option<ScriptLang>,
    tag: Option<String>,
+    dedicated_worker: Option<bool>,
 }

 #[derive(Deserialize)]
@@ -1531,6 +1477,7 @@ struct PreviewFlow {
    path: Option<String>,
    args: Option<Box<JsonRawValue>>,
    tag: Option<String>,
+    restarted_from: Option<RestartedFrom>,
 }

 pub struct QueryOrBody<D>(pub Option<D>);
@@ -1622,7 +1569,7 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) ->

 #[cfg(feature = "enterprise")]
 pub async fn check_license_key_valid() -> error::Result<()> {
-    use crate::LICENSE_KEY_VALID;
+    use windmill_common::ee::LICENSE_KEY_VALID;

    let valid = *LICENSE_KEY_VALID.read().await;
    if !valid {
@@ -1686,6 +1633,82 @@ pub async fn run_flow_by_path(
    Ok((StatusCode::CREATED, uuid.to_string()))
 }

+pub async fn restart_flow(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
+    Path((w_id, job_id, step_id, branch_or_iteration_n)): Path<(
+        String,
+        Uuid,
+        String,
+        Option<usize>,
+    )>,
+    Query(run_query): Query<RunJobQuery>,
+) -> error::Result<(StatusCode, String)> {
+    #[cfg(not(feature = "enterprise"))]
+    {
+        return Err(Error::BadRequest(
+            "Restarting a flow is a feature only available in enterprise version".to_string(),
+        ));
+    }
+
+    #[cfg(feature = "enterprise")]
+    check_license_key_valid().await?;
+
+    let completed_job = sqlx::query_as::<_, CompletedJob>(
+        "SELECT * from completed_job WHERE id = $1 and workspace_id = $2",
+    )
+    .bind(job_id)
+    .bind(&w_id)
+    .fetch_optional(&db)
+    .await?
+    .with_context(|| "Unable to find completed job with the given job UUID")?;
+
+    let flow_path = completed_job
+        .script_path
+        .with_context(|| "No flow path set for completed flow job")?;
+    check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
+
+    let push_args = completed_job
+        .args
+        .map(|json| PushArgs { args: json.clone(), extra: json.0 });
+
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
+
+    let (uuid, tx) = push(
+        &db,
+        tx,
+        &w_id,
+        JobPayload::RestartedFlow {
+            completed_job_id: job_id,
+            step_id: step_id,
+            branch_or_iteration_n: branch_or_iteration_n,
+        },
+        push_args,
+        &authed.username,
+        &authed.email,
+        username_to_permissioned_as(&authed.username),
+        scheduled_for,
+        None,
+        run_query.parent_job,
+        run_query.parent_job,
+        run_query.job_id,
+        false,
+        false,
+        None,
+        !run_query.invisible_to_owner.unwrap_or(false),
+        Some(completed_job.tag),
+        None,
+        None,
+        completed_job.priority,
+    )
+    .await?;
+    tx.commit().await?;
+    Ok((StatusCode::CREATED, uuid.to_string()))
+}
+
 pub async fn run_job_by_path(
    authed: ApiAuthed,
    Extension(db): Extension<DB>,
@@ -2240,6 +2263,7 @@ async fn run_preview_job(
                concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
                concurrency_time_window_s: None, // TODO(gbouv): same as above
                cache_ttl: None,
+                dedicated_worker: preview.dedicated_worker,
            }),
        },
        preview.args.unwrap_or_default(),
@@ -2324,7 +2348,7 @@ async fn add_batch_jobs(

            let mut uuids: Vec<Uuid> = Vec::new();
            let payload = if let Some(ref fv) = batch_info.flow_value {
-                JobPayload::RawFlow { value: fv.clone(), path: None }
+                JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None }
            } else {
                if let Some(path) = batch_info.path.as_ref() {
                    JobPayload::Flow(path.to_string())
@@ -2442,7 +2466,11 @@ async fn run_preview_flow_job(
        &db,
        tx,
        &w_id,
-        JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path },
+        JobPayload::RawFlow {
+            value: raw_flow.value,
+            path: raw_flow.path,
+            restarted_from: raw_flow.restarted_from,
+        },
        raw_flow.args.unwrap_or_default(),
        &authed.username,
        &authed.email,
@@ -2769,7 +2797,8 @@ async fn get_completed_job<'a>(
        .await?;

    let job = not_found_if_none(job_o, "Completed Job", id.to_string())?;
-    Ok(CompletedJob::from_row(&job)?.into_response())
+    let response = Json(CompletedJob::from_row(&job)?).into_response();
+    Ok(response)
 }

 #[derive(FromRow)]
@@ -2900,5 +2929,6 @@ async fn delete_completed_job<'a>(
    .await?;

    tx.commit().await?;
-    Ok(CompletedJob::from_row(&job)?.into_response())
+    let response = Json(CompletedJob::from_row(&job)?).into_response();
+    Ok(response)
 }
--- a/backend/windmill-api/src/lib.rs
+++ b/backend/windmill-api/src/lib.rs
@@ -7,6 +7,7 @@
 */

 use crate::db::ApiAuthed;
+use crate::embeddings::load_embeddings_db;
 use crate::oauth2::AllClients;
 use crate::saml::{SamlSsoLogin, ServiceProviderExt};
 use crate::scim::has_scim_token;
@@ -103,10 +104,6 @@ lazy_static::lazy_static! {
        connects: HashMap::new(),
        slack: None
    }));
-
-    pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
-    pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-    pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 }

 pub async fn run_server(
@@ -115,6 +112,7 @@ pub async fn run_server(
    addr: SocketAddr,
    mut rx: tokio::sync::broadcast::Receiver<()>,
    port_tx: tokio::sync::oneshot::Sender<u16>,
+    server_mode: bool,
 ) -> anyhow::Result<()> {
    if let Some(mut rsmq) = rsmq.clone() {
        for tag in ALL_TAGS.read().await.iter() {
@@ -164,6 +162,12 @@ pub async fn run_server(
    #[cfg(not(feature = "enterprise"))]
    let sp_extension = (ServiceProviderExt(), SamlSsoLogin(None));

+    let embeddings_db = if server_mode {
+        Some(load_embeddings_db(&db))
+    } else {
+        None
+    };
+
    // build our application with a route
    let app = Router::new()
        .nest(
@@ -195,7 +199,11 @@ pub async fn run_server(
                        )
                        .nest("/variables", variables::workspaced_service())
                        .nest("/workspaces", workspaces::workspaced_service())
-                        .nest("/openai", openai::workspaced_service()),
+                        .nest("/openai", openai::workspaced_service())
+                        .nest(
+                            "/embeddings",
+                            embeddings::workspaced_service(embeddings_db.clone()),
+                        ),
                )
                .nest("/workspaces", workspaces::global_service())
                .nest(
@@ -211,7 +219,10 @@ pub async fn run_server(
                .nest("/flows", flows::global_service())
                .nest("/apps", apps::global_service().layer(cors.clone()))
                .nest("/schedules", schedule::global_service())
-                .nest("/embeddings", embeddings::global_service(&db))
+                .nest(
+                    "/embeddings",
+                    embeddings::global_service(embeddings_db.clone()),
+                )
                .route_layer(from_extractor::<ApiAuthed>())
                .route_layer(from_extractor::<users::Tokened>())
                .nest("/jobs", jobs::global_root_service())
@@ -322,12 +333,15 @@ async fn ee_license() -> &'static str {

 #[cfg(feature = "enterprise")]
 async fn ee_license() -> String {
+    use windmill_common::ee::LICENSE_KEY_ID;
+
    LICENSE_KEY_ID.read().await.clone()
 }

 async fn openapi() -> &'static str {
    include_str!("../openapi-deref.yaml")
 }
+
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
    db::migrate(db).await?;
    Ok(())
--- a/backend/windmill-api/src/oauth2.rs
+++ b/backend/windmill-api/src/oauth2.rs
@@ -56,8 +56,6 @@ use windmill_queue::PushIsolationLevel;

 use std::{fs, str};

-pub const WORKSPACE_SLACK_BOT_TOKEN_PATH: &str = "f/slack_bot/bot_token";
-
 pub fn global_service() -> Router {
    Router::new()
        .route("/login/:client", get(login))
--- a/backend/windmill-api/src/schedule.rs
+++ b/backend/windmill-api/src/schedule.rs
@@ -8,7 +8,9 @@

 use crate::{
    db::{ApiAuthed, DB},
+    settings::set_global_setting_internal,
    users::maybe_refresh_folders,
+    utils::require_super_admin,
 };
 use axum::{
    extract::{Extension, Path, Query},
@@ -40,6 +42,7 @@ pub fn workspaced_service() -> Router {
        .route("/update/*path", post(edit_schedule))
        .route("/delete/*path", delete(delete_schedule))
        .route("/setenabled/*path", post(set_enabled))
+        .route("/setdefaulthandler", post(set_default_error_handler))
 }

 pub fn global_service() -> Router {
@@ -62,6 +65,19 @@ pub struct NewSchedule {
    pub on_recovery: Option<String>,
    pub on_recovery_times: Option<i32>,
    pub on_recovery_extra_args: Option<serde_json::Value>,
+    pub ws_error_handler_muted: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ErrorOrRecoveryHandler {
+    pub handler_type: String, // 'error' or 'recovery'
+    pub override_existing: bool,
+
+    pub path: String,
+    pub extra_args: Option<serde_json::Value>,
+    pub number_of_occurence: Option<i32>,
+    pub number_of_occurence_exact: Option<bool>,
+    pub workspace_handler_muted: Option<bool>,
 }

 async fn check_path_conflict<'c>(
@@ -103,15 +119,6 @@ async fn create_schedule(
        ));
    }

-    #[cfg(not(feature = "enterprise"))]
-    if ns.on_failure.is_some()
-        && ns.on_failure.as_ref().unwrap() == "script/hub/2431/slack/schedule-error-handler-slack"
-    {
-        return Err(Error::BadRequest(
-            "Slack error handler is only available in enterprise version".to_string(),
-        ));
-    }
-
    #[cfg(not(feature = "enterprise"))]
    if ns.on_failure_times.is_some() && ns.on_failure_times.unwrap() > 1 {
        return Err(Error::BadRequest(
@@ -136,7 +143,7 @@ async fn create_schedule(
    let schedule = sqlx::query_as!(
        Schedule,
        "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
-         is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *",
+         is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, ws_error_handler_muted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING *",
        w_id,
        ns.path,
        ns.schedule,
@@ -154,6 +161,7 @@ async fn create_schedule(
        ns.on_recovery,
        ns.on_recovery_times,
        ns.on_recovery_extra_args,
+        ns.ws_error_handler_muted.unwrap_or(false),
    )
    .fetch_one(&mut tx)
    .await
@@ -215,8 +223,8 @@ async fn edit_schedule(
    clear_schedule(tx.transaction_mut(), path, is_flow, &w_id).await?;
    let schedule = sqlx::query_as!(
        Schedule,
-        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10 WHERE path \
-         = $11 AND workspace_id = $12 RETURNING *",
+        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10, ws_error_handler_muted = $11 
+        WHERE path  = $12 AND workspace_id = $13 RETURNING *",
        es.schedule,
        es.timezone,
        es.args,
@@ -227,6 +235,7 @@ async fn edit_schedule(
        es.on_recovery,
        es.on_recovery_times,
        es.on_recovery_extra_args,
+        es.ws_error_handler_muted.unwrap_or(false),
        path,
        w_id,
    )
@@ -317,6 +326,7 @@ pub struct ScheduleWJobs {
    pub on_recovery: Option<String>,
    pub on_recovery_times: Option<i32>,
    pub on_recovery_extra_args: Option<serde_json::Value>,
+    pub ws_error_handler_muted: bool,
    pub jobs: Option<Vec<serde_json::Value>>,
 }

@@ -481,6 +491,77 @@ async fn delete_schedule(
    Ok(format!("schedule {} deleted", path))
 }

+async fn set_default_error_handler(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    Json(payload): Json<ErrorOrRecoveryHandler>,
+) -> Result<()> {
+    require_super_admin(&db, &authed.email).await?;
+    let (key, value) = match payload.handler_type.as_str() {
+        "error" => {
+            let key = format!("default_error_handler_{}", w_id);
+            let value = serde_json::json!({
+                "wsErrorHandlerMuted": payload.workspace_handler_muted,
+                "errorHandlerPath": payload.path,
+                "errorHandlerExtraArgs": payload.extra_args,
+                "failedTimes": payload.number_of_occurence,
+                "failedExact": payload.number_of_occurence_exact,
+            });
+            Ok((key, value))
+        }
+        "recovery" => {
+            let key = format!("default_recovery_handler_{}", w_id);
+            let value = serde_json::json!({
+                "recoveryHandlerPath": payload.path,
+                "recoveryHandlerExtraArgs": payload.extra_args,
+                "recoveredTimes": payload.number_of_occurence,
+            });
+            Ok((key, value))
+        }
+        _ => Err(Error::BadRequest(
+            "handler_type must be either 'error' or 'recovery'".to_string(),
+        )),
+    }?;
+
+    set_global_setting_internal(&db, key, value).await?;
+
+    if payload.override_existing {
+        match payload.handler_type.as_str() {
+            "error" => {
+                sqlx::query!(
+                    "UPDATE schedule SET ws_error_handler_muted = $1, on_failure = $2, on_failure_extra_args = $3, on_failure_times = $4, on_failure_exact = $5 WHERE workspace_id = $6",
+                    payload.workspace_handler_muted,
+                    payload.path,
+                    payload.extra_args,
+                    payload.number_of_occurence,
+                    payload.number_of_occurence_exact,
+                    w_id,
+                )
+                .execute(&db)
+                .await?;
+                Ok(())
+            }
+            "recovery" => {
+                sqlx::query!(
+                    "UPDATE schedule SET on_recovery = $1, on_recovery_extra_args = $2, on_recovery_times = $3 WHERE workspace_id = $4",
+                    payload.path,
+                    payload.extra_args,
+                    payload.number_of_occurence,
+                    w_id,
+                )
+                .execute(&db)
+                .await?;
+                Ok(())
+            }
+            _ => Err(Error::BadRequest(
+                "handler_type must be either 'error' or 'recovery'".to_string(),
+            )),
+        }?;
+    }
+    Ok(())
+}
+
 async fn check_flow_conflict<'c>(
    tx: &mut Transaction<'c, Postgres>,
    w_id: &str,
@@ -519,6 +600,7 @@ pub struct EditSchedule {
    pub on_recovery: Option<String>,
    pub on_recovery_times: Option<i32>,
    pub on_recovery_extra_args: Option<serde_json::Value>,
+    pub ws_error_handler_muted: Option<bool>,
 }

 pub async fn clear_schedule<'c>(
--- a/backend/windmill-api/src/settings.rs
+++ b/backend/windmill-api/src/settings.rs
@@ -12,6 +12,7 @@ use crate::{
    db::{ApiAuthed, DB},
    ee::validate_license_key,
    utils::require_super_admin,
+    HTTP_CLIENT,
 };

 use axum::{
@@ -24,7 +25,7 @@ use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
 use serde::Deserialize;
 use tokio::time::timeout;
 use windmill_common::{
-    error::{self, to_anyhow, JsonResult},
+    error::{self, to_anyhow, JsonResult, Result},
    global_settings::ENV_SETTINGS,
    server::Smtp,
 };
@@ -38,6 +39,7 @@ pub fn global_service() -> Router {
        )
        .route("/test_smtp", post(test_email))
        .route("/test_license_key", post(test_license_key))
+        .route("/send_stats", post(send_stats))
 }

 #[derive(Deserialize)]
@@ -123,25 +125,32 @@ pub async fn set_global_setting(
    Json(value): Json<Value>,
 ) -> error::Result<()> {
    require_super_admin(&db, &authed.email).await?;
-    match value.value {
+    set_global_setting_internal(&db, key, value.value).await
+}
+
+pub async fn set_global_setting_internal(
+    db: &DB,
+    key: String,
+    value: serde_json::Value,
+) -> error::Result<()> {
+    match value {
        serde_json::Value::Null => {
-            delete_global_setting(&db, &key).await?;
+            delete_global_setting(db, &key).await?;
        }
        serde_json::Value::String(x) if x.is_empty() => {
-            delete_global_setting(&db, &key).await?;
+            delete_global_setting(db, &key).await?;
        }
        v => {
            sqlx::query!(
-            "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
-            key,
-            v
-        )
-        .execute(&db)
-        .await?;
+                "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
+                key,
+                v
+            )
+            .execute(db)
+            .await?;
            tracing::info!("Set global setting {} to {}", key, v);
        }
    };
-
    Ok(())
 }

@@ -150,7 +159,9 @@ pub async fn get_global_setting(
    authed: ApiAuthed,
    Path(key): Path<String>,
 ) -> JsonResult<serde_json::Value> {
-    require_super_admin(&db, &authed.email).await?;
+    if !key.starts_with("default_error_handler_") && !key.starts_with("default_recovery_handler_") {
+        require_super_admin(&db, &authed.email).await?;
+    }
    let value = sqlx::query!("SELECT value FROM global_settings WHERE name = $1", key)
        .fetch_optional(&db)
        .await?
@@ -158,3 +169,10 @@ pub async fn get_global_setting(

    Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null)))
 }
+
+pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    windmill_common::stats::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
+
+    Ok("Sent stats".to_string())
+}
--- a/backend/windmill-api/src/webhook_util.rs
+++ b/backend/windmill-api/src/webhook_util.rs
@@ -109,13 +109,13 @@ impl WebhookShared {
                                }
                            };
                            if let Some(url) = webhook_opt {
-                                let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
+                                let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
                                let _ = client.post(url).json(&message).send().await;
                                timer.map(|x| x.stop_and_record());
                            }
                        },
                        Some(WebhookPayload::InstanceEvent(event)) => {
-                            if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
+                            if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
                            let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
                            if let Err(e) = r {
                                tracing::error!("Error sending instance event: {}", e);
--- a/backend/windmill-api/src/workers.rs
+++ b/backend/windmill-api/src/workers.rs
@@ -39,6 +39,7 @@ struct WorkerPing {
    jobs_executed: i32,
    custom_tags: Option<Vec<String>>,
    worker_group: String,
+    wm_version: String,
 }

 #[derive(Serialize, Deserialize)]
@@ -57,7 +58,7 @@ async fn list_worker_pings(

    let rows = sqlx::query_as!(
        WorkerPing,
-        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
+        "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group, wm_version FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
        per_page as i64,
        offset as i64
    )
--- a/backend/windmill-api/src/workspaces.rs
+++ b/backend/windmill-api/src/workspaces.rs
@@ -21,7 +21,6 @@ use crate::{
    variables::build_crypt,
    webhook_util::{InstanceEvent, WebhookShared}
 };
-use crate::oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH;
 #[cfg(feature = "enterprise")]
 use axum::response::Redirect;
 use axum::{
@@ -36,6 +35,7 @@ use chrono::Utc;
 use magic_crypt::MagicCryptTrait;
 #[cfg(feature = "enterprise")]
 use stripe::CustomerId;
+use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::db::UserDB;
 use windmill_common::schedule::Schedule;
@@ -46,6 +46,7 @@ use windmill_common::{
    scripts::{Schema, Script, ScriptLang},
    utils::{paginate, rd_string, require_admin, Pagination},
    variables::ExportableListableVariable,
+    oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH,
 };
 use windmill_queue::QueueTransaction;

@@ -133,6 +134,7 @@ pub struct WorkspaceSettings {
    pub code_completion_enabled: bool,
    pub error_handler: Option<String>,
    pub error_handler_extra_args: Option<serde_json::Value>,
+    pub error_handler_muted_on_cancel: Option<bool>,
 }

 #[derive(FromRow, Serialize, Debug)]
@@ -245,6 +247,7 @@ pub struct NewWorkspaceUser {
 pub struct EditErrorHandler {
    pub error_handler: Option<String>,
    pub error_handler_extra_args: Option<serde_json::Value>,
+    pub error_handler_muted_on_cancel: Option<bool>,
 }

 async fn list_pending_invites(
@@ -543,7 +546,8 @@ async fn run_slack_message_test_job(
    let tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
    let (uuid, tx) = windmill_queue::handle_on_failure(
        &db, 
-        tx, 
+        tx,
+        Uuid::parse_str("00000000-0000-0000-0000-000000000000")?,
        "slack_message_test", 
        "slack_message_test", 
        false, 
@@ -553,9 +557,7 @@ async fn run_slack_message_test_job(
        0, 
        Utc::now(), 
        Some(json!(extra_args)),
-        authed.username.as_str(), 
        authed.email.as_str(), 
-        username_to_permissioned_as(authed.username.as_str()),
        None, // Note: we could mark it as high priority to return result quickly to the user
    ).await?;
    tx.commit().await?;
@@ -810,14 +812,6 @@ async fn edit_error_handler(
 ) -> Result<String> {
    require_admin(is_admin, &username)?;

-    #[cfg(not(feature = "enterprise"))]
-    if ee.error_handler.as_ref().is_some_and(|val|  val == "script/hub/2431/slack/schedule-error-handler-slack")
-    {
-        return Err(Error::BadRequest(
-            "Slack error handler is only available in enterprise version".to_string(),
-        ));
-    }
-
    let mut tx = db.begin().await?;
    
    sqlx::query_as!(
@@ -833,9 +827,10 @@ async fn edit_error_handler(

    if let Some(error_handler) = &ee.error_handler {
        sqlx::query!(
-            "UPDATE workspace_settings SET error_handler = $1, error_handler_extra_args = $2 WHERE workspace_id = $3",
+            "UPDATE workspace_settings SET error_handler = $1, error_handler_extra_args = $2, error_handler_muted_on_cancel = $3 WHERE workspace_id = $4",
            error_handler,
            ee.error_handler_extra_args,
+            ee.error_handler_muted_on_cancel.unwrap_or(false),
            &w_id
        )
        .execute(&mut *tx)
@@ -1440,6 +1435,29 @@ struct ScriptMetadata {
    is_template: bool,
    lock: Vec<String>,
    kind: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    envs: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    concurrent_limit: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    concurrency_time_window_s: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    cache_ttl: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    dedicated_worker: Option<bool>,
+    #[serde(skip_serializing_if = "is_none_or_false")]
+    ws_error_handler_muted: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    priority: Option<i16>,    
+    #[serde(skip_serializing_if = "Option::is_none")]
+    tag: Option<String>,    
+}
+
+pub fn is_none_or_false(val: &Option<bool>) -> bool {
+    match val {
+        Some(val) => !val,
+        None => true,
+    }
 }

 enum ArchiveImpl {
@@ -1629,6 +1647,15 @@ async fn tarball_workspace(
                is_template: script.is_template,
                kind: script.kind.to_string(),
                lock,
+                envs: script.envs,
+                concurrent_limit: script.concurrent_limit,
+                concurrency_time_window_s: script.concurrency_time_window_s,
+                cache_ttl: script.cache_ttl,
+                dedicated_worker: script.dedicated_worker,
+                ws_error_handler_muted: script.ws_error_handler_muted,
+                priority: script.priority,
+                tag: script.tag,
+                
            };
            let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
            archive
--- a/backend/windmill-common/Cargo.toml
+++ b/backend/windmill-common/Cargo.toml
@@ -17,7 +17,7 @@ tracing_init = [
    "dep:tracing-subscriber",
 ]
 flamegraph = ["dep:tracing-flame"]
-
+loki = ["dep:tracing-loki"]

 [lib]
 name = "windmill_common"
@@ -48,3 +48,4 @@ itertools.workspace = true
 regex.workspace = true
 git-version.workspace = true
 cron.workspace = true
+tracing-loki = { version = "^0", optional = true }
--- a/backend/windmill-common/src/ee.rs
+++ b/backend/windmill-common/src/ee.rs
@@ -0,0 +1,8 @@
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+}
--- a/backend/windmill-common/src/flow_status.rs
+++ b/backend/windmill-common/src/flow_status.rs
@@ -34,6 +34,8 @@ pub struct FlowStatus {
    pub retry: RetryStatus,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub approval_conditions: Option<ApprovalConditions>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub restarted_from: Option<RestartedFrom>,
 }

 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
@@ -50,6 +52,14 @@ pub struct ApprovalConditions {
    pub user_groups_required: Vec<String>,
 }

+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
+#[serde(default)]
+pub struct RestartedFrom {
+    pub flow_job_id: Uuid,
+    pub step_id: String,
+    pub branch_or_iteration_n: Option<usize>,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Iterator {
    pub index: usize,
@@ -179,6 +189,13 @@ impl FlowStatusModule {
            FlowStatusModule::Failure { id, .. } => id.clone(),
        }
    }
+
+    pub fn is_failure(&self) -> bool {
+        match self {
+            FlowStatusModule::Failure { .. } => true,
+            _ => false,
+        }
+    }
 }

 impl FlowStatus {
@@ -202,6 +219,7 @@ impl FlowStatus {
                },
            },
            retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
+            restarted_from: None,
        }
    }

--- a/backend/windmill-common/src/global_settings.rs
+++ b/backend/windmill-common/src/global_settings.rs
@@ -9,6 +9,9 @@ pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
 pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
 pub const UNIQUE_ID_SETTING: &str = "uid";
 pub const DISABLE_STATS_SETTING: &str = "disable_stats";
+pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics";
+pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics";
+pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir";

 pub const ENV_SETTINGS: [&str; 54] = [
    "DISABLE_NSJAIL",
--- a/backend/windmill-common/src/jobs.rs
+++ b/backend/windmill-common/src/jobs.rs
@@ -7,7 +7,7 @@ use uuid::Uuid;

 use crate::{
    error::{self, Error},
-    flow_status::FlowStatus,
+    flow_status::{FlowStatus, RestartedFrom},
    flows::FlowValue,
    get_latest_deployed_hash_for_path,
    scripts::{ScriptHash, ScriptLang},
@@ -186,6 +186,82 @@ impl Default for QueuedJob {
    }
 }

+#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
+pub struct CompletedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub started_at: chrono::DateTime<chrono::Utc>,
+    pub duration_ms: i64,
+    pub success: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_hash: Option<ScriptHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_path: Option<String>,
+    pub args: Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result: Option<sqlx::types::Json<Box<RawValue>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub logs: Option<String>,
+    pub deleted: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_code: Option<String>,
+    pub canceled: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_by: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_reason: Option<String>,
+    pub job_kind: JobKind,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flow_status: Option<sqlx::types::Json<Box<RawValue>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_flow: Option<sqlx::types::Json<Box<RawValue>>>,
+    pub is_flow_step: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language: Option<ScriptLang>,
+    pub is_skipped: bool,
+    pub email: String,
+    pub visible_to_owner: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
+    pub tag: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub priority: Option<i16>,
+}
+
+impl CompletedJob {
+    pub fn json_result(&self) -> Option<serde_json::Value> {
+        self.result
+            .as_ref()
+            .map(|r| serde_json::from_str(r.get()).ok())
+            .flatten()
+    }
+
+    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
+        self.raw_flow
+            .as_ref()
+            .and_then(|v| serde_json::from_str::<FlowValue>((**v).get()).ok())
+    }
+
+    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
+        self.flow_status
+            .as_ref()
+            .and_then(|v| serde_json::from_str::<FlowStatus>((**v).get()).ok())
+    }
+}
+
+#[derive(sqlx::FromRow)]
+pub struct BranchResults<'a> {
+    pub result: &'a RawValue,
+    pub id: Uuid,
+}
+
 #[derive(Debug, Clone)]
 pub enum JobPayload {
    ScriptHub {
@@ -216,9 +292,15 @@ pub enum JobPayload {
        version: i64,
    },
    Flow(String),
+    RestartedFlow {
+        completed_job_id: Uuid,
+        step_id: String,
+        branch_or_iteration_n: Option<usize>,
+    },
    RawFlow {
        value: FlowValue,
        path: Option<String>,
+        restarted_from: Option<RestartedFrom>,
    },
    Identity,
    Noop,
@@ -233,6 +315,7 @@ pub struct RawCode {
    pub concurrent_limit: Option<i32>,
    pub concurrency_time_window_s: Option<i32>,
    pub cache_ttl: Option<i32>,
+    pub dedicated_worker: Option<bool>,
 }

 type Tag = String;
@@ -330,8 +413,3 @@ pub async fn get_payload_tag_from_prefixed_path(
    };
    Ok((payload, tag))
 }
-
-#[derive(Clone)]
-pub struct Metrics {
-    pub worker_execution_failed: prometheus::IntCounter,
-}
--- a/backend/windmill-common/src/lib.rs
+++ b/backend/windmill-common/src/lib.rs
@@ -6,7 +6,10 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use std::{net::SocketAddr, sync::Arc};
+use std::{
+    net::SocketAddr,
+    sync::{atomic::AtomicBool, Arc},
+};

 use error::Error;
 use scripts::ScriptLang;
@@ -14,6 +17,7 @@ use sqlx::{Pool, Postgres};

 pub mod apps;
 pub mod db;
+pub mod ee;
 pub mod error;
 pub mod external_ip;
 pub mod flow_status;
@@ -38,17 +42,26 @@ pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
 pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;

 lazy_static::lazy_static! {
-    pub static ref METRICS_ADDR: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+    pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
+    .ok()
+    .and_then(|s| s.parse::<u16>().ok())
+    .unwrap_or(8001);
+
+    pub static ref METRICS_ADDR: SocketAddr = std::env::var("METRICS_ADDR")
    .ok()
    .map(|s| {
        s.parse::<bool>()
-            .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+            .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT))))
            .or_else(|_| s.parse::<SocketAddr>().map(Some))
    })
    .transpose().ok()
    .flatten()
-    .flatten();
-    pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
+    .flatten()
+    .unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT)));
+
+    pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
+    pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
+
    pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
    pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 }
@@ -92,9 +105,14 @@ pub async fn serve_metrics(
 ) -> JoinHandle<()> {
    use std::sync::atomic::Ordering;

-    use axum::{routing::get, Router};
+    use axum::{
+        routing::{get, post},
+        Router,
+    };
    use hyper::StatusCode;
-    let router = Router::new().route("/metrics", get(metrics));
+    let router = Router::new()
+        .route("/metrics", get(metrics))
+        .route("/reset", post(reset));

    let router = if ready_worker_endpoint {
        router.route(
@@ -112,6 +130,7 @@ pub async fn serve_metrics(
    };

    tokio::spawn(async move {
+        tracing::info!("Serving metrics at: {addr}");
        if let Err(e) = axum::Server::bind(&addr)
            .serve(router.into_make_service())
            .with_graceful_shutdown(async {
@@ -132,6 +151,10 @@ async fn metrics() -> Result<String, Error> {
        .map_err(anyhow::Error::from)?)
 }

+async fn reset() -> () {
+    todo!()
+}
+
 #[cfg(feature = "sqlx")]
 pub async fn connect_db(server_mode: bool) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
    use anyhow::Context;
--- a/backend/windmill-common/src/oauth2.rs
+++ b/backend/windmill-common/src/oauth2.rs
@@ -10,3 +10,5 @@ use hmac::Hmac;
 use sha2::Sha256;

 pub type HmacSha256 = Hmac<Sha256>;
+
+pub const WORKSPACE_SLACK_BOT_TOKEN_PATH: &str = "f/slack_bot/bot_token";
--- a/backend/windmill-common/src/schedule.rs
+++ b/backend/windmill-common/src/schedule.rs
@@ -33,6 +33,7 @@ pub struct Schedule {
    pub on_recovery: Option<String>,
    pub on_recovery_times: Option<i32>,
    pub on_recovery_extra_args: Option<serde_json::Value>,
+    pub ws_error_handler_muted: bool,
 }

 pub fn schedule_to_user(path: &str) -> String {
--- a/backend/windmill-common/src/scripts.rs
+++ b/backend/windmill-common/src/scripts.rs
@@ -171,6 +171,8 @@ pub struct Script {
    pub ws_error_handler_muted: Option<bool>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub priority: Option<i16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_ttl: Option<i32>,
 }

 #[derive(Serialize)]
--- a/backend/windmill-common/src/stats.rs
+++ b/backend/windmill-common/src/stats.rs
@@ -2,11 +2,11 @@ use std::str::FromStr;

 use crate::{
    error::{to_anyhow, Result},
-    global_settings::{DISABLE_STATS_SETTING, UNIQUE_ID_SETTING},
-    utils::GIT_VERSION,
+    global_settings::DISABLE_STATS_SETTING,
+    scripts::ScriptLang,
+    utils::{get_uid, GIT_VERSION},
    DB,
 };
-
 use chrono::Utc;
 use cron::Schedule;

@@ -74,24 +74,54 @@ pub async fn schedule_stats(db: &DB, instance_name: String, http_client: &reqwes
    });
 }

+#[derive(Debug, sqlx::FromRow, serde::Serialize)]
+struct JobsUsage {
+    language: Option<ScriptLang>,
+    total_duration: i64,
+    count: i64,
+}
+
 pub async fn send_stats(
    instance_name: &String,
    http_client: &reqwest::Client,
    db: &DB,
 ) -> Result<()> {
-    let uid = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        UNIQUE_ID_SETTING
+    let uid = get_uid(db).await?;
+
+    let jobs_usage = sqlx::query_as::<_, JobsUsage>(
+        "SELECT language, COUNT(*) as count, SUM(duration_ms)::BIGINT as total_duration FROM completed_job GROUP BY language",
    )
-    .fetch_one(db)
+    .fetch_all(db)
    .await?;

-    let uid = serde_json::from_value::<String>(uid).map_err(to_anyhow)?;
+    let login_type_usage =
+        sqlx::query!("SELECT login_type, COUNT(*) FROM password GROUP BY login_type")
+            .fetch_all(db)
+            .await?
+            .into_iter()
+            .map(|r| {
+                serde_json::json!({
+                    "login_type": r.login_type,
+                    "count": r.count.unwrap_or(0),
+                })
+            })
+            .collect::<Vec<serde_json::Value>>();
+
+    let workers_usage = sqlx::query!(
+        "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'"
+    )
+    .fetch_one(db)
+    .await?
+    .count
+    .unwrap_or(0);

    let payload = serde_json::json!({
        "uid": uid,
        "version": GIT_VERSION,
        "instance_name": instance_name,
+        "jobs_usage": jobs_usage,
+        "login_type_usage": login_type_usage,
+        "workers_usage": workers_usage,
    });

    let request = http_client
--- a/backend/windmill-common/src/tracing_init.rs
+++ b/backend/windmill-common/src/tracing_init.rs
@@ -41,6 +41,15 @@ pub fn initialize_tracing() {

    let ts_base = tracing_subscriber::registry().with(env_filter);

+    #[cfg(feature = "loki")]
+    let ts_base = {
+        let (layer, task) = tracing_loki::builder()
+            .build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap())
+            .expect("build loki url");
+        tokio::spawn(task);
+        ts_base.with(layer)
+    };
+
    match json_fmt {
        true => ts_base.with(json_layer().flatten_event(true)).init(),
        false => ts_base
--- a/backend/windmill-common/src/utils.rs
+++ b/backend/windmill-common/src/utils.rs
@@ -6,6 +6,7 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use crate::ee::LICENSE_KEY_ID;
 use crate::error::{to_anyhow, Error, Result};
 use crate::global_settings::UNIQUE_ID_SETTING;
 use crate::DB;
@@ -103,13 +104,7 @@ pub async fn http_get_from_hub(
    query_params: Option<Vec<(&str, String)>>,
    db: &Pool<Postgres>,
 ) -> Result<reqwest::Response> {
-    let uid = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        UNIQUE_ID_SETTING
-    )
-    .fetch_optional(db)
-    .await?
-    .map(|v| serde_json::from_value::<String>(v));
+    let uid = get_uid(db).await;

    let mut request = http_client.get(url).header(
        "Accept",
@@ -120,12 +115,10 @@ pub async fn http_get_from_hub(
        },
    );

-    if let Some(uid) = uid {
-        if let Ok(uid) = uid {
-            request = request.header("X-uid", uid);
-        } else {
-            tracing::info!("Invalid uid in global settings: {}", uid.err().unwrap())
-        }
+    if let Ok(uid) = uid {
+        request = request.header("X-uid", uid);
+    } else {
+        tracing::info!("No valid uid found: {}", uid.err().unwrap())
    }

    if let Some(query_params) = query_params {
@@ -152,3 +145,20 @@ pub fn calculate_hash(s: &str) -> String {
    hasher.update(s);
    format!("{:x}", hasher.finalize())
 }
+
+pub async fn get_uid(db: &DB) -> Result<String> {
+    let mut uid = LICENSE_KEY_ID.read().await.clone();
+
+    if uid == "" {
+        let uid_value = sqlx::query_scalar!(
+            "SELECT value FROM global_settings WHERE name = $1",
+            UNIQUE_ID_SETTING
+        )
+        .fetch_one(db)
+        .await?;
+
+        uid = serde_json::from_value::<String>(uid_value).map_err(to_anyhow)?;
+    }
+
+    Ok(uid)
+}
--- a/backend/windmill-common/src/worker.rs
+++ b/backend/windmill-common/src/worker.rs
@@ -135,13 +135,14 @@ pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db:
        )
    };
    sqlx::query!(
-        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
+        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
        worker_instance,
        worker_name,
        ip,
        tags.as_slice(),
        *WORKER_GROUP,
-        dw
+        dw,
+        crate::utils::GIT_VERSION
    )
    .execute(db)
    .await
@@ -208,10 +209,14 @@ pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
        .worker_tags
        .or_else(|| {
            if let Some(ref dedicated_worker) = dedicated_worker.as_ref() {
-                Some(vec![format!(
+                let mut dedi_tags = vec![format!(
                    "{}:{}",
                    dedicated_worker.workspace_id, dedicated_worker.path
-                )])
+                )];
+                if std::env::var("ADD_FLOW_TAG").is_ok() {
+                    dedi_tags.push("flow".to_string());
+                }
+                Some(dedi_tags)
            } else {
                std::env::var("WORKER_TAGS")
                    .ok()
@@ -219,6 +224,7 @@ pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
            }
        })
        .unwrap_or_else(|| DEFAULT_TAGS.clone());
+
    let mut priority_tags_sorted: Vec<PriorityTags> = Vec::new();
    let priority_tags_map = config.priority_tags.unwrap_or_else(HashMap::new);
    if priority_tags_map.len() > 0 {
--- a/backend/windmill-queue/src/jobs.rs
+++ b/backend/windmill-queue/src/jobs.rs
--- a/backend/windmill-queue/src/schedule.rs
+++ b/backend/windmill-queue/src/schedule.rs
@@ -72,7 +72,15 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
    }

    let (payload, tag) = if schedule.is_flow {
-        (JobPayload::Flow(schedule.script_path), None)
+        let tag = sqlx::query_scalar!(
+            "SELECT tag from flow WHERE path = $1 and workspace_id = $2",
+            &schedule.script_path,
+            &schedule.workspace_id,
+        )
+        .fetch_optional(&mut tx)
+        .await?
+        .flatten();
+        (JobPayload::Flow(schedule.script_path), tag)
    } else {
        let (
            hash,
--- a/backend/windmill-worker/Cargo.toml
+++ b/backend/windmill-worker/Cargo.toml
@@ -73,6 +73,7 @@ urlencoding.workspace = true
 nix.workspace = true
 bytes.workspace = true
 reqwest.workspace = true
+hex.workspace = true

 [build-dependencies]
 deno_fetch.workspace = true
--- a/backend/windmill-worker/loader.py
+++ b/backend/windmill-worker/loader.py
@@ -26,7 +26,7 @@ class WindmillFinder(MetaPathFinder):
        l = len(splitted)
        if l <= 2:
            return ModuleSpec(name, WindmillLoader(name))
-        elif l == 3:
+        elif l > 2:
            script_path = "/".join(splitted)
            import requests

@@ -44,12 +44,9 @@ class WindmillFinder(MetaPathFinder):
                    f.write(r.text)
                return ModuleSpec(name, SourceFileLoader(name, fullpath))
            else:
-                print(r.text, r.status_code)
-                raise ImportError(f"Script {script_path} not found")
-        else:
-            raise ImportError(
-                "Import can only be done at the top level of a folder or user space"
-            )
+                # raise ImportError(f"Script {script_path} not found")
+                return ModuleSpec(name, WindmillLoader(name))
+


 sys.meta_path.append(WindmillFinder)
--- a/backend/windmill-worker/src/bash_executor.rs
+++ b/backend/windmill-worker/src/bash_executor.rs
@@ -6,6 +6,7 @@ use serde_json::{json, value::RawValue};
 use sqlx::types::Json;
 use tokio::process::Command;
 use windmill_common::{error::Error, jobs::QueuedJob, worker::to_raw_value};
+use windmill_queue::CanceledBy;

 const BIN_BASH: &str = "/bin/bash";
 const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
@@ -28,6 +29,7 @@ lazy_static::lazy_static! {
 pub async fn handle_bash_job(
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
@@ -115,6 +117,7 @@ pub async fn handle_bash_job(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        !*DISABLE_NSJAIL,
        worker_name,
@@ -160,6 +163,7 @@ fn raw_to_string(x: &str) -> String {
 pub async fn handle_powershell_job(
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
@@ -180,7 +184,7 @@ pub async fn handle_powershell_job(
            job.args.as_ref()
        };

-        let args_owned = windmill_parser_bash::parse_bash_sig(&content)?
+        let args_owned = windmill_parser_bash::parse_powershell_sig(&content)?
            .args
            .iter()
            .map(|arg| {
@@ -253,6 +257,7 @@ pub async fn handle_powershell_job(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        !*DISABLE_NSJAIL,
        worker_name,
--- a/backend/windmill-worker/src/bun_executor.rs
+++ b/backend/windmill-worker/src/bun_executor.rs
@@ -5,9 +5,10 @@ use itertools::Itertools;
 use regex::Regex;
 use serde_json::value::RawValue;
 use uuid::Uuid;
+use windmill_queue::CanceledBy;

 #[cfg(feature = "enterprise")]
-use crate::{common::build_envs_map, JobCompleted};
+use crate::common::build_envs_map;

 use crate::{
    common::{
@@ -26,7 +27,7 @@ use tokio::{
 use tokio::io::AsyncReadExt;

 #[cfg(feature = "enterprise")]
-use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::sync::mpsc::Receiver;

 #[cfg(feature = "enterprise")]
 use windmill_common::variables;
@@ -53,6 +54,7 @@ lazy_static::lazy_static! {
 pub async fn gen_lockfile(
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job_id: &Uuid,
    w_id: &str,
    db: &sqlx::Pool<sqlx::Postgres>,
@@ -100,6 +102,7 @@ pub async fn gen_lockfile(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child_process,
        false,
        worker_name,
@@ -144,6 +147,7 @@ pub async fn gen_lockfile(
    install_lockfile(
        logs,
        mem_peak,
+        canceled_by,
        job_id,
        w_id,
        db,
@@ -180,6 +184,7 @@ pub async fn gen_lockfile(
 pub async fn install_lockfile(
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job_id: &Uuid,
    w_id: &str,
    db: &sqlx::Pool<sqlx::Postgres>,
@@ -202,6 +207,7 @@ pub async fn install_lockfile(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child_process,
        false,
        worker_name,
@@ -236,6 +242,7 @@ pub async fn handle_bun_job(
    requirements_o: Option<String>,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
@@ -279,6 +286,7 @@ pub async fn handle_bun_job(
            install_lockfile(
                logs,
                mem_peak,
+                canceled_by,
                &job.id,
                &job.workspace_id,
                db,
@@ -302,6 +310,7 @@ pub async fn handle_bun_job(
            let _ = gen_lockfile(
                logs,
                mem_peak,
+                canceled_by,
                &job.id,
                &job.workspace_id,
                db,
@@ -489,6 +498,7 @@ plugin(p)
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        false,
        worker_name,
@@ -526,6 +536,8 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String
    return bun_envs;
 }

+#[cfg(feature = "enterprise")]
+use crate::{dedicated_worker::handle_dedicated_process, JobCompletedSender};
 #[cfg(feature = "enterprise")]
 use std::sync::Arc;

@@ -541,14 +553,13 @@ pub async fn start_worker(
    w_id: &str,
    script_path: &str,
    token: &str,
-    job_completed_tx: Sender<JobCompleted>,
+    job_completed_tx: JobCompletedSender,
    jobs_rx: Receiver<Arc<QueuedJob>>,
    killpill_rx: tokio::sync::broadcast::Receiver<()>,
 ) -> Result<()> {
-    use crate::dedicated_worker::handle_dedicated_process;
-
    let mut logs = "".to_string();
    let mut mem_peak: i32 = 0;
+    let mut canceled_by: Option<CanceledBy> = None;
    let _ = write_file(job_dir, "main.ts", inner_content).await?;
    let common_bun_proc_envs: HashMap<String, String> =
        get_common_bun_proc_envs(&base_internal_url).await;
@@ -577,34 +588,45 @@ pub async fn start_worker(
        let _ = write_file(job_dir, "package.json", &splitted[0]).await?;
        let lockb = splitted[1];
        if lockb != EMPTY_FILE {
-            let _ = write_file_binary(
+            let has_trusted_deps = &splitted[0].contains("trustedDependencies");
+
+            if !has_trusted_deps {
+                let _ = write_file_binary(
+                    job_dir,
+                    "bun.lockb",
+                    &base64::engine::general_purpose::STANDARD
+                        .decode(&splitted[1])
+                        .map_err(|_| {
+                            error::Error::InternalErr("Could not decode bun.lockb".to_string())
+                        })?,
+                )
+                .await?;
+            }
+
+            install_lockfile(
+                &mut logs,
+                &mut mem_peak,
+                &mut canceled_by,
+                &Uuid::nil(),
+                &w_id,
+                db,
                job_dir,
-                "bun.lockb",
-                &base64::engine::general_purpose::STANDARD
-                    .decode(&splitted[1])
-                    .map_err(|_| {
-                        error::Error::InternalErr("Could not decode bun.lockb".to_string())
-                    })?,
+                worker_name,
+                common_bun_proc_envs.clone(),
            )
            .await?;
+            if !has_trusted_deps {
+                remove_dir_all(format!("{}/node_modules", job_dir)).await?;
+            }
+            tracing::info!("dedicated worker requirements installed: {reqs}");
        }
-        install_lockfile(
-            &mut logs,
-            &mut mem_peak,
-            &Uuid::nil(),
-            &w_id,
-            db,
-            job_dir,
-            worker_name,
-            common_bun_proc_envs.clone(),
-        )
-        .await?;
    } else if !*DISABLE_NSJAIL {
        let trusted_deps = get_trusted_deps(inner_content);
        logs.push_str("\n\n--- BUN INSTALL ---\n");
        let _ = gen_lockfile(
            &mut logs,
            &mut mem_peak,
+            &mut canceled_by,
            &Uuid::nil(),
            &w_id,
            db,
--- a/backend/windmill-worker/src/common.rs
+++ b/backend/windmill-worker/src/common.rs
@@ -17,6 +17,7 @@ use windmill_common::{
 };

 use anyhow::Result;
+use windmill_queue::CanceledBy;

 use std::{
    borrow::Borrow,
@@ -211,7 +212,9 @@ pub async fn transform_json_value(
                .get_variable_value(path)
                .await
                .map(|x| json!(x))
-                .map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`")))
+                .map_err(|e| {
+                    Error::NotFound(format!("Variable {path} not found for `{name}`: {e}"))
+                })
        }
        Value::String(y) if y.starts_with("$res:") => {
            let path = y.strip_prefix("$res:").unwrap();
@@ -226,7 +229,9 @@ pub async fn transform_json_value(
                    Some(job.id.to_string()),
                )
                .await
-                .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))
+                .map_err(|e| {
+                    Error::NotFound(format!("Resource {path} not found for `{name}`: {e}"))
+                })
        }
        Value::String(y) if y.starts_with("$") => {
            let flow_path = if let Some(uuid) = job.parent_job {
@@ -418,6 +423,7 @@ pub async fn handle_child(
    db: &Pool<Postgres>,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by_ref: &mut Option<CanceledBy>,
    mut child: Child,
    nsjail: bool,
    worker_name: &str,
@@ -479,16 +485,22 @@ pub async fn handle_child(
                    if current_mem > *mem_peak {
                        *mem_peak = current_mem
                    }
-                    tracing::info!("{job_id} in {_w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
-                    if sqlx::query_scalar!("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled", *mem_peak, job_id)
+                    tracing::info!("{worker_name}/{job_id} in {_w_id} still running.  mem: {current_mem}kB, peak mem: {mem_peak}kB");
+                    let (canceled, canceled_by, canceled_reason) = sqlx::query_as::<_, (bool, Option<String>, Option<String>)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason")
+                        .bind(*mem_peak)
+                        .bind(job_id)
                        .fetch_optional(&db)
                        .await
-                        .map(|v| Some(true) == v)
-                        .unwrap_or_else(|err| {
-                            tracing::error!(%job_id, %err, "error checking cancelation for job {job_id}: {err}");
-                            false
+                        .unwrap_or_else(|e| {
+                            tracing::error!(%e, "error updating job {job_id}: {e}");
+                            Some((false, None, None))
                        })
-                    {
+                        .unwrap_or((false, None, None));
+                    if canceled {
+                        canceled_by_ref.replace(CanceledBy {
+                            username: canceled_by.clone(),
+                            reason: canceled_reason.clone(),
+                        });
                        break;
                    }
                },
@@ -689,7 +701,7 @@ pub async fn handle_child(

    let (wait_result, _) = tokio::join!(wait_on_child, lines);

-    tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms, mem_peak: {:?}", start.elapsed().as_millis(), mem_peak);
+    tracing::info!(%job_id, "child process '{child_name}' for {worker_name}/{job_id} took {}ms, mem_peak: {:?}", start.elapsed().as_millis(), mem_peak);
    match wait_result {
        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
@@ -776,7 +788,7 @@ fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
    }
 }

-pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> i64 {
+pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>) -> String {
    if let Some(vs) = v {
        let mut dh = DefaultHasher::new();
        let hm = &vs.0;
@@ -784,9 +796,9 @@ pub fn hash_args(v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>)
            k.hash(&mut dh);
            hm.get(k).unwrap().get().hash(&mut dh);
        }
-        dh.finish() as i64
+        hex::encode(dh.finish().to_be_bytes())
    } else {
-        0
+        "empty_args".to_string()
    }
 }

--- a/backend/windmill-worker/src/dedicated_worker.rs
+++ b/backend/windmill-worker/src/dedicated_worker.rs
@@ -8,18 +8,20 @@ use tokio::{
    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
    process::Command,
 };
-use windmill_common::{error, jobs::QueuedJob, variables};
+use windmill_common::{error, jobs::QueuedJob, variables, worker::to_raw_value};

 use std::{collections::VecDeque, process::Stdio, sync::Arc};

 use anyhow::Context;

-use crate::{common::start_child_process, JobCompleted, MAX_BUFFERED_DEDICATED_JOBS};
+use crate::{
+    common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
+};

 use futures::{future, Future};
 use std::{collections::HashMap, task::Poll};

-use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::sync::mpsc::Receiver;

 fn conditional_polling<T>(
    fut: impl Future<Output = T>,
@@ -50,7 +52,7 @@ pub async fn handle_dedicated_process(
    common_bun_proc_envs: HashMap<String, String>,
    args: Vec<&str>,
    mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
-    job_completed_tx: Sender<JobCompleted>,
+    job_completed_tx: JobCompletedSender,
    token: &str,
    mut jobs_rx: Receiver<Arc<QueuedJob>>,
 ) -> std::result::Result<(), error::Error> {
@@ -80,8 +82,15 @@ pub async fn handle_dedicated_process(
        .take()
        .expect("child did not have a handle to stdout");

+    let stderr = child
+        .stderr
+        .take()
+        .expect("child did not have a handle to stderr");
+
    let mut reader = BufReader::new(stdout).lines();

+    let mut err_reader = BufReader::new(stderr).lines();
+
    let mut stdin = child
        .stdin
        .take()
@@ -113,6 +122,14 @@ pub async fn handle_dedicated_process(
                    tracing::info!("Could not write end message to stdin: {e:?}")
                }
            },
+            line = err_reader.next_line() => {
+                if let Some(line) = line.expect("line is ok") {
+                    tracing::error!("dedicated worker process stderr: {:?}", line);
+                } else {
+                    tracing::info!("dedicated worker process exited");
+                    break;
+                }
+            },
            line = reader.next_line() => {
                // j += 1;

@@ -121,11 +138,16 @@ pub async fn handle_dedicated_process(
                        tracing::info!("dedicated worker process started");
                        continue;
                    }
-                    tracing::debug!("processed job");
+                    tracing::debug!("processed job: {line}");

-                    let result = serde_json::from_str(&line).expect("json is ok");
                    let job: Arc<QueuedJob> = jobs.pop_front().expect("pop");
-                    job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), mem_peak: 0, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap();
+                    match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line) {
+                        Ok(result) => job_completed_tx.send(JobCompleted { job , result, logs: "".to_string(), mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string() }).await.unwrap(),
+                        Err(e) => {
+                            tracing::error!("Could not deserialize job result `{line}`: {e:?}");
+                            job_completed_tx.send(JobCompleted { job , result: to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")})), logs: "".to_string(), mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string() }).await.unwrap();
+                        },
+                    };
                } else {
                    tracing::info!("dedicated worker process exited");
                    break;
--- a/backend/windmill-worker/src/deno_executor.rs
+++ b/backend/windmill-worker/src/deno_executor.rs
@@ -3,6 +3,7 @@ use std::{collections::HashMap, process::Stdio};
 use itertools::Itertools;
 use serde_json::value::RawValue;
 use uuid::Uuid;
+use windmill_queue::CanceledBy;

 use crate::{
    common::{
@@ -75,6 +76,7 @@ pub async fn generate_deno_lock(
    code: &str,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job_dir: &str,
    db: &sqlx::Pool<sqlx::Postgres>,
    w_id: &str,
@@ -120,6 +122,7 @@ pub async fn generate_deno_lock(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child_process,
        false,
        worker_name,
@@ -142,6 +145,7 @@ pub async fn handle_deno_job(
    requirements_o: Option<String>,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
@@ -330,6 +334,7 @@ run().catch(async (e) => {{
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        false,
        worker_name,
--- a/backend/windmill-worker/src/go_executor.rs
+++ b/backend/windmill-worker/src/go_executor.rs
@@ -14,6 +14,7 @@ use windmill_common::{
    utils::calculate_hash,
 };
 use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE};
+use windmill_queue::CanceledBy;

 use crate::{
    common::{
@@ -35,6 +36,7 @@ lazy_static::lazy_static! {
 pub async fn handle_go_job(
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
@@ -79,6 +81,7 @@ pub async fn handle_go_job(
            inner_content,
            logs,
            mem_peak,
+            canceled_by,
            job_dir,
            db,
            true,
@@ -195,6 +198,7 @@ func Run(req Req) (interface{{}}, error){{
            db,
            logs,
            mem_peak,
+            canceled_by,
            build_go_process,
            false,
            worker_name,
@@ -275,6 +279,7 @@ func Run(req Req) (interface{{}}, error){{
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        !*DISABLE_NSJAIL,
        worker_name,
@@ -313,6 +318,7 @@ pub async fn install_go_dependencies(
    code: &str,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job_dir: &str,
    db: &sqlx::Pool<sqlx::Postgres>,
    non_dep_job: bool,
@@ -336,6 +342,7 @@ pub async fn install_go_dependencies(
            db,
            logs,
            mem_peak,
+            canceled_by,
            child_process,
            false,
            worker_name,
@@ -400,6 +407,7 @@ pub async fn install_go_dependencies(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child_process,
        false,
        worker_name,
--- a/backend/windmill-worker/src/js_eval.rs
+++ b/backend/windmill-worker/src/js_eval.rs
@@ -86,10 +86,10 @@ pub async fn eval_timeout(
        }
    }

-    if expr.starts_with("flow_input.") {
+    if expr.starts_with("flow_input.") || expr.starts_with("flow_input[") {
        if let Some(ref flow_input) = flow_input {
            for (k,v) in flow_input.iter() {
-                if &format!("flow_input.{k}") == &expr {
+                if &format!("flow_input.{k}") == &expr || &format!("flow_input[\"{k}\"]") == &expr  {
                    // tracing::error!("FLOW_INPUT");
                    return Ok(v.clone())
                }
@@ -98,12 +98,17 @@ pub async fn eval_timeout(
    }

    let p_id = by_id.as_ref().map(|x| format!("results.{}", x.previous_id));
+    let p_id2 = by_id.as_ref().map(|x| format!("results[\"{}\"]", x.previous_id));

    if p_id.is_some() && transform_context.contains_key("previous_result") && &expr == p_id.as_ref().unwrap() {
        // tracing::error!("PREVIOUS_RESULT");
        return Ok(transform_context.get("previous_result").unwrap().as_ref().clone())
    }

+    if p_id2.is_some() && transform_context.contains_key("previous_result") && &expr == p_id2.as_ref().unwrap() {
+        return Ok(transform_context.get("previous_result").unwrap().as_ref().clone())
+    }
+
    if by_id.is_some() && authed_client.is_some() {
        if let Some(x) = RE_FULL.captures(&expr).and_then(|x| x.get(1).map(|y| y.as_str())) {
            // tracing::error!("{:?}", x.split(".").collect::<Vec<_>>());
@@ -161,7 +166,7 @@ pub async fn eval_timeout(
                .map(|x| x.clone())
                .collect_vec();

-            if !context_keys.contains(&"previous_result".to_string()) && (p_id.is_some() && expr.contains(p_id.as_ref().unwrap()))  || expr.contains("error") {
+            if !context_keys.contains(&"previous_result".to_string()) && (p_id.is_some() && expr.contains(p_id.as_ref().unwrap()))  || expr.contains("error") || (p_id2.is_some() && expr.contains(p_id2.as_ref().unwrap())) {
                context_keys.push("previous_result".to_string());
            }
            let has_flow_input = expr.contains("flow_input");
@@ -173,7 +178,14 @@ pub async fn eval_timeout(
            {
                let op_state = js_runtime.op_state();
                let mut op_state = op_state.borrow_mut();
-                op_state.put(OptAuthedClient(authed_client.clone()));
+                let mut client = authed_client.clone();
+                if let Some(client) = client.as_mut() {
+                    client.force_client = Some(reqwest::ClientBuilder::new()
+                    .user_agent("windmill/beta")
+                    .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
+                    .build().unwrap());
+                }
+                op_state.put(OptAuthedClient(client));
                op_state.put(TransformContext {
                    flow_input: if has_flow_input { flow_input } else { None },
                    envs: transform_context
@@ -230,7 +242,7 @@ fn replace_with_await(expr: String, fn_name: &str) -> String {
    s
 }
 lazy_static! {
-    static ref RE: Regex = Regex::new(r"(?m)(?P<r>results\.(?:[a-z]|[A-Z]|_|[1-9])+)").unwrap();
+    static ref RE: Regex = Regex::new(r#"(?m)(?P<r>results(?:(?:\.(?:[a-z]|[A-Z]|_|[1-9])+)|(?:\[\".*?\"\])))"#).unwrap();
    static ref RE_FULL: Regex = Regex::new(r"(?m)^results((?:\.(?:(?:[a-z]|[A-Z]|_|[1-9])+))+)$").unwrap();

 }
--- a/backend/windmill-worker/src/python_executor.rs
+++ b/backend/windmill-worker/src/python_executor.rs
@@ -17,6 +17,7 @@ use windmill_common::{
    worker::WORKER_CONFIG,
    DB,
 };
+use windmill_queue::CanceledBy;

 lazy_static::lazy_static! {
    static ref PYTHON_PATH: String =
@@ -67,6 +68,7 @@ pub async fn pip_compile(
    requirements: &str,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    job_dir: &str,
    db: &Pool<Postgres>,
    worker_name: &str,
@@ -132,6 +134,7 @@ pub async fn pip_compile(
        db,
        logs,
        mem_peak,
+        canceled_by,
        child_process,
        false,
        worker_name,
@@ -169,6 +172,7 @@ pub async fn handle_python_job(
    job: &QueuedJob,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClientBackgroundTask,
    inner_content: &String,
@@ -189,6 +193,7 @@ pub async fn handle_python_job(
        worker_dir,
        logs,
        mem_peak,
+        canceled_by,
    )
    .await?;

@@ -352,6 +357,7 @@ mount {{
        db,
        logs,
        mem_peak,
+        canceled_by,
        child,
        !*DISABLE_NSJAIL,
        worker_name,
@@ -502,6 +508,7 @@ async fn handle_python_deps(
    worker_dir: &str,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
 ) -> error::Result<Vec<String>> {
    create_dependencies_dir(job_dir).await;

@@ -532,6 +539,7 @@ async fn handle_python_deps(
                    &requirements,
                    logs,
                    mem_peak,
+                    canceled_by,
                    job_dir,
                    db,
                    worker_name,
@@ -555,6 +563,7 @@ async fn handle_python_deps(
            w_id,
            logs,
            mem_peak,
+            canceled_by,
            db,
            worker_name,
            job_dir,
@@ -571,6 +580,7 @@ pub async fn handle_python_reqs(
    w_id: &str,
    logs: &mut String,
    mem_peak: &mut i32,
+    canceled_by: &mut Option<CanceledBy>,
    db: &sqlx::Pool<sqlx::Postgres>,
    worker_name: &str,
    job_dir: &str,
@@ -721,6 +731,7 @@ pub async fn handle_python_reqs(
            db,
            logs,
            mem_peak,
+            canceled_by,
            child,
            false,
            worker_name,
@@ -752,11 +763,11 @@ pub async fn handle_python_reqs(

 #[cfg(feature = "enterprise")]
 use std::sync::Arc;
-#[cfg(feature = "enterprise")]
-use tokio::sync::mpsc::Sender;

 #[cfg(feature = "enterprise")]
-use crate::{common::build_envs_map, dedicated_worker::handle_dedicated_process, JobCompleted};
+use crate::JobCompletedSender;
+#[cfg(feature = "enterprise")]
+use crate::{common::build_envs_map, dedicated_worker::handle_dedicated_process};
 #[cfg(feature = "enterprise")]
 use tokio::sync::mpsc::Receiver;
 #[cfg(feature = "enterprise")]
@@ -774,12 +785,13 @@ pub async fn start_worker(
    w_id: &str,
    script_path: &str,
    token: &str,
-    job_completed_tx: Sender<JobCompleted>,
+    job_completed_tx: JobCompletedSender,
    jobs_rx: Receiver<Arc<QueuedJob>>,
    killpill_rx: tokio::sync::broadcast::Receiver<()>,
 ) -> error::Result<()> {
    let mut logs = "".to_string();
    let mut mem_peak: i32 = 0;
+    let mut canceled_by: Option<CanceledBy> = None;
    let context = variables::get_reserved_variables(
        w_id,
        &token,
@@ -809,6 +821,7 @@ pub async fn start_worker(
        job_dir,
        &mut logs,
        &mut mem_peak,
+        &mut canceled_by,
    )
    .await?;

@@ -827,8 +840,17 @@ pub async fn start_worker(
    ) = prepare_wrapper(job_dir, inner_content, script_path).await?;

    {
-        // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
-        // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud
+        let indented_transforms = transforms
+            .lines()
+            .map(|x| format!("    {}", x))
+            .collect::<Vec<String>>()
+            .join("\n");
+        let indented_spread = spread
+            .lines()
+            .map(|x| format!("    {}", x))
+            .collect::<Vec<String>>()
+            .join("\n");
+
        let wrapper_content: String = format!(
            r#"
 import json
@@ -852,8 +874,8 @@ sys.stdout.write('start\n')
 for line in sys.stdin:
    kwargs = json.loads(line, strict=False)
    args = {{}}
-    {transforms}
-    {spread}
+{indented_transforms}
+{indented_spread}
    for k, v in list(args.items()):
        if v == '<function call>':
            del args[k]
--- a/backend/windmill-worker/src/worker.rs
+++ b/backend/windmill-worker/src/worker.rs
--- a/backend/windmill-worker/src/worker_flow.rs
+++ b/backend/windmill-worker/src/worker_flow.rs
@@ -6,7 +6,10 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use std::collections::hash_map::DefaultHasher;
 use std::collections::HashMap;
+use std::hash::Hash;
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::time::Duration;

@@ -27,7 +30,8 @@ use windmill_common::flow_status::{
    ApprovalConditions, FlowStatusModuleWParent, Iterator, JobResult,
 };
 use windmill_common::jobs::{
-    script_hash_to_tag_and_limits, script_path_to_payload, JobPayload, Metrics, QueuedJob, RawCode,
+    script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob,
+    RawCode,
 };
 use windmill_common::worker::to_raw_value;
 use windmill_common::{
@@ -39,8 +43,8 @@ use windmill_common::{
    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend},
 };
 use windmill_queue::{
-    add_completed_job, add_completed_job_error, handle_maybe_scheduled_job, PushIsolationLevel,
-    WrappedError,
+    add_completed_job, add_completed_job_error, handle_maybe_scheduled_job, CanceledBy,
+    PushIsolationLevel, WrappedError,
 };

 type DB = sqlx::Pool<sqlx::Postgres>;
@@ -59,12 +63,12 @@ pub async fn update_flow_status_after_job_completion<
    w_id: &str,
    success: bool,
    result: &'a RawValue,
-    metrics: Option<Metrics>,
    unrecoverable: bool,
    same_worker_tx: Sender<Uuid>,
    worker_dir: &str,
    stop_early_override: Option<bool>,
    rsmq: Option<R>,
+    worker_name: &str,
 ) -> error::Result<()> {
    // this is manual tailrecursion because async_recursion blows up the stack
    let mut rec = update_flow_status_after_job_completion_internal(
@@ -75,13 +79,13 @@ pub async fn update_flow_status_after_job_completion<
        w_id,
        success,
        result,
-        metrics.clone(),
        unrecoverable,
        same_worker_tx.clone(),
        worker_dir,
        stop_early_override,
        false,
        rsmq.clone(),
+        worker_name,
    )
    .await?;
    while let Some(nrec) = rec {
@@ -93,13 +97,13 @@ pub async fn update_flow_status_after_job_completion<
            w_id,
            nrec.success,
            nrec.result.as_ref(),
-            metrics.clone(),
            false,
            same_worker_tx.clone(),
            worker_dir,
            nrec.stop_early_override,
            nrec.skip_error_handler,
            rsmq.clone(),
+            worker_name,
        )
        .await
        {
@@ -113,13 +117,13 @@ pub async fn update_flow_status_after_job_completion<
                    w_id,
                    false,
                    &to_raw_value(&Json(&WrappedError { error: json!(e.to_string()) })),
-                    metrics.clone(),
                    true,
                    same_worker_tx.clone(),
                    worker_dir,
                    nrec.stop_early_override,
                    nrec.skip_error_handler,
                    rsmq.clone(),
+                    worker_name,
                )
                .await?
            }
@@ -136,12 +140,6 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
    skip_error_handler: bool,
 }

-#[derive(FromRow)]
-pub struct BranchResults<'a> {
-    pub result: &'a RawValue,
-    pub id: Uuid,
-}
-
 #[derive(FromRow)]
 pub struct SkipIfStopped {
    pub skip_if_stopped: Option<bool>,
@@ -161,13 +159,13 @@ pub async fn update_flow_status_after_job_completion_internal<
    w_id: &str,
    mut success: bool,
    result: &'a RawValue,
-    metrics: Option<Metrics>,
    unrecoverable: bool,
    same_worker_tx: Sender<Uuid>,
    worker_dir: &str,
    stop_early_override: Option<bool>,
    skip_error_handler: bool,
    rsmq: Option<R>,
+    worker_name: &str,
 ) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
    let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult, is_failure_step) = {
        // tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
@@ -507,35 +505,7 @@ pub async fn update_flow_status_after_job_completion_internal<
        let nresult = match &new_status {
            Some(FlowStatusModule::Success { flow_jobs: Some(jobs), .. })
            | Some(FlowStatusModule::Failure { flow_jobs: Some(jobs), .. }) => {
-                let results = sqlx::query(
-                    "
-                  SELECT result, id
-                    FROM completed_job
-                   WHERE id = ANY($1)
-                     AND workspace_id = $2
-                    ",
-                )
-                .bind(jobs.as_slice())
-                .bind(w_id)
-                .fetch_all(&mut tx)
-                .await?
-                .into_iter()
-                .map(|r| {
-                    let br = BranchResults::from_row(&r).unwrap();
-                    (br.id, br.result.to_owned())
-                })
-                .collect::<HashMap<_, _>>();
-
-                let results = jobs
-                    .iter()
-                    .map(|j| {
-                        results.get(j).ok_or_else(|| {
-                            Error::InternalErr(format!("missing job result for {}", j))
-                        })
-                    })
-                    .collect::<Result<Vec<_>, _>>()?;
-
-                to_raw_value(&results)
+                retrieve_flow_jobs_results(db, w_id, jobs).await?
            }
            _ => result.to_owned(),
        };
@@ -637,9 +607,13 @@ pub async fn update_flow_status_after_job_completion_internal<
                &flow_job,
                logs,
                0,
+                Some(CanceledBy {
+                    username: flow_job.canceled_by.clone(),
+                    reason: flow_job.canceled_reason.clone(),
+                }),
                canceled_job_to_result(&flow_job),
-                metrics.clone(),
                rsmq.clone(),
+                worker_name,
            )
            .await?;
        } else {
@@ -647,7 +621,17 @@ pub async fn update_flow_status_after_job_completion_internal<
                let cached_res_path = {
                    let args_hash = hash_args(&flow_job.args);
                    let flow_path = flow_job.script_path();
-                    format!("{flow_path}/flow/cache/{args_hash}")
+                    let version_hash = if let Some(rc) = flow_job.raw_flow.as_ref() {
+                        use std::hash::Hasher;
+                        let mut s = DefaultHasher::new();
+                        serde_json::to_string(&rc.0)
+                            .unwrap_or_default()
+                            .hash(&mut s);
+                        format!("flow_{}", hex::encode(s.finish().to_be_bytes()))
+                    } else {
+                        "flow_unknown".to_string()
+                    };
+                    format!("{flow_path}/cache/{version_hash}/{args_hash}")
                };

                save_in_cache(db, &flow_job, cached_res_path, &nresult).await;
@@ -662,6 +646,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                    Json(&nresult),
                    logs,
                    0,
+                    None,
                    rsmq.clone(),
                )
                .await?;
@@ -678,6 +663,7 @@ pub async fn update_flow_status_after_job_completion_internal<
                    ),
                    logs,
                    0,
+                    None,
                    rsmq.clone(),
                )
                .await?;
@@ -689,10 +675,11 @@ pub async fn update_flow_status_after_job_completion_internal<
            &flow_job,
            db,
            client,
-            nresult.to_owned(),
+            Some(nresult.to_owned()),
            same_worker_tx.clone(),
            worker_dir,
            rsmq.clone(),
+            worker_name,
        )
        .await
        {
@@ -703,9 +690,10 @@ pub async fn update_flow_status_after_job_completion_internal<
                    &flow_job,
                    "Unexpected error during flow chaining:\n".to_string(),
                    0,
+                    None,
                    e,
-                    metrics.clone(),
                    rsmq.clone(),
+                    worker_name,
                )
                .await;
                true
@@ -715,7 +703,7 @@ pub async fn update_flow_status_after_job_completion_internal<
    };

    if done {
-        if flow_job.same_worker && !*KEEP_JOB_DIR {
+        if flow_job.same_worker && !KEEP_JOB_DIR.load(Ordering::Relaxed) {
            let _ = tokio::fs::remove_dir_all(format!("{worker_dir}/{}", flow_job.id)).await;
        }

@@ -739,6 +727,42 @@ pub async fn update_flow_status_after_job_completion_internal<
    }
 }

+async fn retrieve_flow_jobs_results(
+    db: &DB,
+    w_id: &str,
+    job_uuids: &Vec<Uuid>,
+) -> error::Result<Box<RawValue>> {
+    let results = sqlx::query(
+        "
+      SELECT result, id
+        FROM completed_job
+       WHERE id = ANY($1)
+         AND workspace_id = $2
+        ",
+    )
+    .bind(job_uuids.as_slice())
+    .bind(w_id)
+    .fetch_all(db)
+    .await?
+    .into_iter()
+    .map(|r| {
+        let br = BranchResults::from_row(&r).unwrap();
+        (br.id, br.result.to_owned())
+    })
+    .collect::<HashMap<_, _>>();
+
+    let results = job_uuids
+        .iter()
+        .map(|j| {
+            results
+                .get(j)
+                .ok_or_else(|| Error::InternalErr(format!("missing job result for {}", j)))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    Ok(to_raw_value(&results))
+}
+
 fn get_module(flow_job: &QueuedJob, module_index: Option<usize>) -> Option<FlowModule> {
    let raw_flow = flow_job.parse_raw_flow();
    if let Some(raw_flow) = raw_flow {
@@ -833,7 +857,7 @@ async fn compute_bool_from_expr(
 ) -> error::Result<bool> {
    let mut context = HashMap::with_capacity(if resumes.is_some() { 7 } else { 3 });
    context.insert("result".to_string(), result.clone());
-    context.insert("previous_result".to_string(), result);
+    context.insert("previous_result".to_string(), result.clone());

    if let Some(resumes) = resumes {
        context.insert("resume".to_string(), resumes.1);
@@ -982,24 +1006,19 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
    flow_job: &QueuedJob,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClient,
-    last_result: Box<RawValue>,
+    last_result: Option<Box<RawValue>>,
    same_worker_tx: Sender<Uuid>,
    worker_dir: &str,
    rsmq: Option<R>,
+    worker_name: &str,
 ) -> anyhow::Result<()> {
-    let value = flow_job
-        .raw_flow
-        .as_ref()
-        .ok_or_else(|| Error::InternalErr(format!("requiring a raw flow value")))?
-        .to_owned();
-    let flow = serde_json::from_str::<FlowValue>((*value.0).get())?;
+    let flow = flow_job
+        .parse_raw_flow()
+        .with_context(|| "Unable to parse flow definition")?;
+    let status = flow_job
+        .parse_flow_status()
+        .with_context(|| "Unable to parse flow status")?;

-    let status: FlowStatus = serde_json::from_str::<FlowStatus>(
-        (*flow_job.flow_status.clone().unwrap_or_default().0).get(),
-    )
-    .with_context(|| format!("parse flow status {}", flow_job.id))?;
-
-    tracing::debug!("handle_flow: {:#?}", flow_job.flow_status);
    push_next_flow_job(
        flow_job,
        status,
@@ -1010,6 +1029,7 @@ pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
        same_worker_tx,
        worker_dir,
        rsmq,
+        worker_name,
    )
    .await?;
    Ok(())
@@ -1050,10 +1070,11 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
    flow: FlowValue,
    db: &sqlx::Pool<sqlx::Postgres>,
    client: &AuthedClient,
-    last_result: Box<RawValue>,
+    last_job_result: Option<Box<RawValue>>,
    same_worker_tx: Sender<Uuid>,
    worker_dir: &str,
    rsmq: Option<R>,
+    worker_name: &str,
 ) -> error::Result<()> {
    let job_root = flow_job
        .root_job
@@ -1089,25 +1110,24 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                // it has to be an empty for loop event
                serde_json::from_str("[]").unwrap()
            },
-            None,
            true,
            same_worker_tx,
            worker_dir,
            None,
            rsmq,
+            worker_name,
        )
        .await;
    }

    let arc_flow_job_args = Arc::new(flow_job_args.clone());
-    let mut arc_result = Arc::new(last_result);

    if i == 0 {
        if let Some(skip_expr) = &flow.skip_expr {
            let skip = compute_bool_from_expr(
                skip_expr.to_string(),
                arc_flow_job_args.clone(),
-                arc_result.clone(),
+                Arc::new(to_raw_value(&json!("{}"))),
                None,
                Some(client),
                None,
@@ -1122,18 +1142,39 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                    flow_job.workspace_id.as_str(),
                    true,
                    serde_json::from_str("\"stopped early\"").unwrap(),
-                    None,
                    true,
                    same_worker_tx,
                    worker_dir,
                    Some(true),
                    rsmq,
+                    worker_name,
                )
                .await;
            }
        }
    }

+    // Compute and initialize last_job_result
+    let arc_last_job_result = if status_module.is_failure() {
+        // if job is being retried, pass the result of its previous failure
+        Arc::new(last_job_result.unwrap_or(to_raw_value(&json!("{}"))))
+    } else if i == 0 {
+        // if it's the first job executed in the flow, pass the flow args
+        Arc::new(to_raw_value(&flow_job.args))
+    } else {
+        // else pass the last job result. Either from the function arg if it's set, or manually fetch it from the previous job
+        // having last_job_result empty can happen either when the job was suspended and is being restarted, or if it's a
+        // flow restart from a specific step
+        if last_job_result.is_some() {
+            Arc::new(last_job_result.unwrap())
+        } else {
+            match get_previous_job_result(db, flow_job.workspace_id.as_str(), &status).await? {
+                None => Arc::new(to_raw_value(&json!("{}"))),
+                Some(previous_job_result) => Arc::new(previous_job_result),
+            }
+        }
+    };
+
    let mut resume_messages: Vec<Box<RawValue>> = vec![];
    let mut approvers: Vec<String> = vec![];

@@ -1197,8 +1238,9 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                        }
                        InputTransform::Javascript { expr } => {
                            let mut context = HashMap::with_capacity(2);
-                            context.insert("result".to_string(), arc_result.clone());
-                            context.insert("previous_result".to_string(), arc_result.clone());
+                            context.insert("result".to_string(), arc_last_job_result.clone());
+                            context
+                                .insert("previous_result".to_string(), arc_last_job_result.clone());

                            let eval_result = serde_json::from_str::<Vec<String>>(
                                eval_timeout(
@@ -1270,20 +1312,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                .execute(&mut *tx)
                .await?;

-                /* If we are woken up after suspending, last_result will be the flow args, but we
-                 * should use the result from the last job */
-                if let FlowStatusModule::WaitingForEvents { .. } = &status_module {
-                    arc_result = Arc::new(
-                        sqlx::query_scalar::<_, Json<Box<RawValue>>>(
-                            "SELECT result FROM completed_job WHERE id = $1",
-                        )
-                        .bind(last)
-                        .fetch_one(&mut *tx)
-                        .await?
-                        .0,
-                    )
-                }
-
                // Remove the approval conditions from the flow status
                sqlx::query(
                    "
@@ -1331,6 +1359,14 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                let skipped = false;
                let logs = "Timed out waiting to be resumed".to_string();
                let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}});
+                let canceled_by = if flow_job.canceled {
+                    Some(CanceledBy {
+                        username: flow_job.canceled_by.clone(),
+                        reason: flow_job.canceled_reason.clone(),
+                    })
+                } else {
+                    None
+                };
                let _uuid = add_completed_job(
                    db,
                    &flow_job,
@@ -1339,6 +1375,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                    Json(&result),
                    logs,
                    0,
+                    canceled_by,
                    rsmq,
                )
                .await?;
@@ -1381,8 +1418,8 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                    InputTransform::Static { value } => Ok(value),
                    InputTransform::Javascript { expr } => {
                        let mut context = HashMap::with_capacity(2);
-                        context.insert("result".to_string(), arc_result.clone());
-                        context.insert("previous_result".to_string(), arc_result.clone());
+                        context.insert("result".to_string(), arc_last_job_result.clone());
+                        context.insert("previous_result".to_string(), arc_last_job_result.clone());

                        serde_json::from_str(
                            eval_timeout(
@@ -1404,6 +1441,12 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                };
                match json_value {
                    Ok(serde_json::Value::Number(n)) => {
+                        if !n.is_u64() {
+                            return Err(Error::ExecutionErr(format!(
+                                "Expected an integer, found: {n}"
+                            )));
+                        }
+
                        n.as_u64().map(|x| from_now(Duration::from_secs(x)))
                    }
                    _ => Err(Error::ExecutionErr(format!(
@@ -1557,7 +1600,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                let by_id = transform_context.as_ref().unwrap();
                transform_input(
                    arc_flow_job_args.clone(),
-                    arc_result.clone(),
+                    arc_last_job_result.clone(),
                    input_transforms,
                    resumes.clone(),
                    resume.clone(),
@@ -1568,8 +1611,10 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                .await
            }
            FlowModuleValue::Identity => serde_json::from_str(
-                &serde_json::to_string(&PreviousResult { previous_result: Some(&arc_result) })
-                    .unwrap(),
+                &serde_json::to_string(&PreviousResult {
+                    previous_result: Some(&arc_last_job_result),
+                })
+                .unwrap(),
            )
            .map_err(|e| error::Error::InternalErr(format!("identity: {e}"))),

@@ -1579,7 +1624,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>

    let next_flow_transform = compute_next_flow_transform(
        arc_flow_job_args.clone(),
-        arc_result.clone(),
+        arc_last_job_result.clone(),
        flow_job,
        &flow,
        transform_context,
@@ -1696,7 +1741,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                        let ctx = get_transform_context(&flow_job, &previous_id, &status).await?;
                        transform_inp = transform_input(
                            Arc::new(hm),
-                            arc_result.clone(),
+                            arc_last_job_result.clone(),
                            input_transforms,
                            resumes.clone(),
                            resume.clone(),
@@ -2008,7 +2053,7 @@ enum NextFlowTransform {

 async fn compute_next_flow_transform(
    arc_flow_job_args: Arc<HashMap<String, Box<RawValue>>>,
-    arc_result: Arc<Box<RawValue>>,
+    arc_last_job_result: Arc<Box<RawValue>>,
    flow_job: &QueuedJob,
    flow: &FlowValue,
    by_id: Option<IdContext>,
@@ -2084,8 +2129,12 @@ async fn compute_next_flow_transform(
        /* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
        FlowModuleValue::ForloopFlow { modules, iterator, parallel, .. } => {
            // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
-            let is_simple =
-                modules.len() == 1 && modules[0].value.is_simple() && flow.failure_module.is_none();
+            let is_simple = modules.len() == 1
+                && modules[0].value.is_simple()
+                && modules[0].sleep.is_none()
+                && modules[0].suspend.is_none()
+                && modules[0].cache_ttl.is_none()
+                && flow.failure_module.is_none();

            let next_loop_status = match status_module {
                FlowStatusModule::WaitingForPriorSteps { .. }
@@ -2100,9 +2149,9 @@ async fn compute_next_flow_transform(
                    let itered_raw = match iterator {
                        InputTransform::Static { value } => to_raw_value(value),
                        InputTransform::Javascript { expr } => {
-                            let mut context = HashMap::with_capacity(3);
-                            context.insert("result".to_string(), arc_result.clone());
-                            context.insert("previous_result".to_string(), arc_result);
+                            let mut context = HashMap::with_capacity(5);
+                            context.insert("result".to_string(), arc_last_job_result.clone());
+                            context.insert("previous_result".to_string(), arc_last_job_result);
                            context.insert("resumes".to_string(), resumes);
                            context.insert("resume".to_string(), resume);
                            context.insert("approvers".to_string(), approvers);
@@ -2144,20 +2193,53 @@ async fn compute_next_flow_transform(
                    flow_jobs: Some(flow_jobs),
                    ..
                } if !*parallel => {
+                    let itered_new = if itered.is_empty() {
+                        // it's possible we need to re-compute the iterator Input Transforms here, in particular if the flow is being restarted inside the loop
+                        let by_id = if let Some(x) = by_id {
+                            x
+                        } else {
+                            get_transform_context(&flow_job, previous_id, &status).await?
+                        };
+                        let itered_raw = match iterator {
+                            InputTransform::Static { value } => to_raw_value(value),
+                            InputTransform::Javascript { expr } => {
+                                let mut context = HashMap::with_capacity(5);
+                                context.insert("result".to_string(), arc_last_job_result.clone());
+                                context.insert("previous_result".to_string(), arc_last_job_result);
+                                context.insert("resumes".to_string(), resumes);
+                                context.insert("resume".to_string(), resume);
+                                context.insert("approvers".to_string(), approvers);
+
+                                eval_timeout(
+                                    expr.to_string(),
+                                    context,
+                                    Some(arc_flow_job_args),
+                                    Some(client),
+                                    Some(by_id),
+                                )
+                                .await?
+                            }
+                        };
+                        serde_json::from_str::<Vec<serde_json::Value>>(itered_raw.get()).map_err(
+                            |not_array| {
+                                Error::ExecutionErr(format!(
+                                    "Expected an array value, found: {not_array}"
+                                ))
+                            },
+                        )?
+                    } else {
+                        itered.clone()
+                    };
                    let (index, next) = index
                        .checked_add(1)
-                        .and_then(|i| itered.get(i).map(|next| (i, next)))
-                        /* we shouldn't get here because update_flow_status_after_job_completion
-                         * should leave this state if there iteration is complete, but also it should
-                         * be reasonable to just enter a completed state instead of failing, similar to
-                         * iterating an empty list above */
+                        .and_then(|i| itered_new.get(i).map(|next| (i, next)))
                        .with_context(|| {
-                            format!("could not iterate index {index} of {itered:?}")
+                            format!("Could not find iteration number {index} restarting inside the for-loop flow. It's possible the itered-array has changed and this value isn't available anymore.")
                        })?;

                    LoopStatus::NextIteration(NextIteration {
                        index,
-                        itered: itered.clone(),
+                        itered: itered_new.clone(),
                        flow_jobs: flow_jobs.clone(),
                        new_args: Iter { index: index as i32, value: next.to_owned() },
                    })
@@ -2192,6 +2274,7 @@ async fn compute_next_flow_transform(
                                priority: None,
                            },
                            path: inner_path,
+                            restarted_from: None,
                        },
                        tag: None,
                    });
@@ -2251,6 +2334,7 @@ async fn compute_next_flow_transform(
                                        priority: None,
                                    },
                                    path: Some(format!("{}/forloop", flow_job.script_path())),
+                                    restarted_from: None,
                                },
                                tag: None,
                            }
@@ -2293,7 +2377,7 @@ async fn compute_next_flow_transform(
                        let pred = compute_bool_from_expr(
                            b.expr.to_string(),
                            arc_flow_job_args.clone(),
-                            arc_result.clone(),
+                            arc_last_job_result.clone(),
                            Some(idcontext.clone()),
                            Some(client),
                            Some((resumes.clone(), resume.clone(), approvers.clone())),
@@ -2348,6 +2432,7 @@ async fn compute_next_flow_transform(
                            flow_job.script_path(),
                            status.step
                        )),
+                        restarted_from: None,
                    },
                    tag: None,
                }),
@@ -2394,6 +2479,7 @@ async fn compute_next_flow_transform(
                                                    flow_job.script_path(),
                                                    i
                                                )),
+                                                restarted_from: None,
                                            },
                                            tag: None,
                                        }
@@ -2457,6 +2543,7 @@ async fn compute_next_flow_transform(
                            flow_job.script_path(),
                            branch_status.branch
                        )),
+                        restarted_from: None,
                    },
                    tag: None,
                }),
@@ -2485,6 +2572,7 @@ fn raw_script_to_payload(
            concurrent_limit: *concurrent_limit,
            concurrency_time_window_s: *concurrency_time_window_s,
            cache_ttl: module.cache_ttl.map(|x| x as i32),
+            dedicated_worker: None,
        }),
        tag: tag.clone(),
    }
@@ -2569,8 +2657,15 @@ fn from_now(duration: Duration) -> chrono::DateTime<chrono::Utc> {
        .unwrap_or(chrono::DateTime::<chrono::Utc>::MAX_UTC)
 }

-/// returns previous module non-zero suspend count and job
+/// returns previous module non-zero suspend count and job, if relevant
 fn needs_resume(flow: &FlowValue, status: &FlowStatus) -> Option<(Suspend, Uuid)> {
+    // for a restarted job, if the restarted step is just after a suspend, don't run the suspend
+    if status.restarted_from.is_some() {
+        let current_step_id = flow.modules.get(status.step as usize)?.id.clone();
+        if status.restarted_from.as_ref().unwrap().step_id == current_step_id {
+            return None;
+        }
+    }
    let prev = usize::try_from(status.step)
        .ok()
        .and_then(|s| s.checked_sub(1))?;
@@ -2591,3 +2686,32 @@ fn needs_resume(flow: &FlowValue, status: &FlowStatus) -> Option<(Suspend, Uuid)
        None
    }
 }
+
+// returns the result of the previous step of a running flow (if the job was successful)
+async fn get_previous_job_result(
+    db: &sqlx::Pool<sqlx::Postgres>,
+    w_id: &str,
+    flow_status: &FlowStatus,
+) -> error::Result<Option<Box<RawValue>>> {
+    let prev = usize::try_from(flow_status.step)
+        .ok()
+        .and_then(|s| s.checked_sub(1))
+        .with_context(|| "No step preceding the current one")?;
+
+    match flow_status.modules.get(prev) {
+        Some(FlowStatusModule::Success { flow_jobs: Some(flow_jobs), .. }) => {
+            Ok(Some(retrieve_flow_jobs_results(db, w_id, flow_jobs).await?))
+        }
+        Some(FlowStatusModule::Success { job, .. }) => Ok(Some(
+            sqlx::query_scalar::<_, Json<Box<RawValue>>>(
+                "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
+            )
+            .bind(job)
+            .bind(w_id)
+            .fetch_one(db)
+            .await?
+            .0,
+        )),
+        _ => Ok(None),
+    }
+}
--- a/benchmarks/lib.ts
+++ b/benchmarks/lib.ts
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";

-export const VERSION = "v1.191.0";
+export const VERSION = "v1.202.1";

 export async function login(email: string, password: string): Promise<string> {
  return await windmill.UserService.login({
--- a/benchmarks/pulumi/index.ts
+++ b/benchmarks/pulumi/index.ts
@@ -198,7 +198,7 @@ const asg = new aws.autoscaling.Group("asg3", {
    version: "$Latest",
  },
  minSize: 0,
-  maxSize: 30,
+  maxSize: 2,
  vpcZoneIdentifiers: [subnetA.id],
  tags: [
    {
@@ -242,7 +242,7 @@ const lb = new awsx.lb.ApplicationLoadBalancer("lb2", {
 //   desiredCount: 2,
 //   taskDefinitionArgs: {
 //     container: {
-//       image: "nginx:latest",
+//       image: "nginx:main@sha256:31ce1955e23ec18963e5d3d7357dc68b1a62f54145d19ceff4054d257066129e",
 //       cpu: 512,
 //       memory: 128,
 //       essential: true,
@@ -261,7 +261,7 @@ db.address.apply((address) => {
    containerDefinitions: JSON.stringify([
      {
        name: "windmill-worker",
-        image: "ghcr.io/windmill-labs/windmill-ee:1.190.3",
+        image: "ghcr.io/windmill-labs/windmill-ee:main@sha256:31ce1955e23ec18963e5d3d7357dc68b1a62f54145d19ceff4054d257066129e",
        cpu: 1024,
        memory: 1800,
        essential: true,
@@ -273,8 +273,8 @@ db.address.apply((address) => {
        ],
        environment: [
          { name: "MODE", value: "worker" },
-          { name: "NUM_WORKERS", value: "1" },
-          { name: "DATABASE_CONNECTIONS", value: "5"},
+          { name: "NUM_WORKERS", value: "5" },
+          { name: "DATABASE_CONNECTIONS", value: "10"},
          { name: "SLEEP_QUEUE", value: "300"},
          // { name: "METRICS_ADDR", value: "true" },
          { name: "RUST_LOG", value: "info" },
@@ -296,6 +296,11 @@ db.address.apply((address) => {
        dockerLabels: {
          PROMETHEUS_EXPORTER_PORT: "8001",
        },
+        portMappings: [
+          {
+            containerPort: 8001,
+          },
+        ],
      },
    ]),
    volumes: [
@@ -314,7 +319,7 @@ db.address.apply((address) => {
    containerDefinitions: JSON.stringify([
      {
        name: "windmill-worker",
-        image: "ghcr.io/windmill-labs/windmill-ee:1.190.3",
+        image: "ghcr.io/windmill-labs/windmill-ee:main@sha256:31ce1955e23ec18963e5d3d7357dc68b1a62f54145d19ceff4054d257066129e",
        cpu: 1024,
        memory: 1800,
        essential: true,
@@ -350,6 +355,11 @@ db.address.apply((address) => {
        dockerLabels: {
          PROMETHEUS_EXPORTER_PORT: "8001",
        },
+        portMappings: [
+          {
+            containerPort: 8001,
+          },
+        ],
      },
    ]),
    volumes: [
@@ -368,7 +378,7 @@ db.address.apply((address) => {
    containerDefinitions: JSON.stringify([
      {
        name: "windmill-server",
-        image: "ghcr.io/windmill-labs/windmill-ee:1.190.3",
+        image: "ghcr.io/windmill-labs/windmill-ee:main@sha256:31ce1955e23ec18963e5d3d7357dc68b1a62f54145d19ceff4054d257066129e",
        cpu: 1024,
        memory: 1024,
        essential: true,
@@ -409,7 +419,7 @@ db.address.apply((address) => {
  const service_server = new aws.ecs.Service("service-server", {
    cluster: cluster.id,
    taskDefinition: server_td.arn,
-    desiredCount: 4,
+    desiredCount: 1,
    forceNewDeployment: true,
    orderedPlacementStrategies: [
      {
@@ -429,7 +439,7 @@ db.address.apply((address) => {
  const service_worker = new aws.ecs.Service("service-worker", {
    cluster: cluster.id,
    taskDefinition: worker_td.arn,
-    desiredCount: 20,
+    desiredCount: 1,
    forceNewDeployment: true,
    orderedPlacementStrategies: [
      {
@@ -442,7 +452,7 @@ db.address.apply((address) => {
  const service_worker2 = new aws.ecs.Service("service-worker-2", {
    cluster: cluster.id,
    taskDefinition: worker_td2.arn,
-    desiredCount: 30,
+    desiredCount: 0,
    forceNewDeployment: true,
    orderedPlacementStrategies: [
      {
--- a/cli/dev.ts
+++ b/cli/dev.ts
@@ -44,14 +44,15 @@ async function dev(opts: GlobalOptions & { filter?: string }) {
      const content = await Deno.readTextFile(cpath);
      const splitted = cpath.split(".");
      const wmPath = splitted[0];
-      const ext = splitted[splitted.length - 1];
+      const len = splitted.length;
+      const ext = splitted[len - 1];
      const lang =
        ext == "py"
          ? "python3"
          : ext == "ts"
-          ? splitted.length > 2 && splitted[splitted.length - 2] == "fetch"
+          ? len > 2 && splitted[len - 2] == "fetch"
            ? "nativets"
-            : splitted.length > 2 && splitted[splitted.length - 2] == "bun"
+            : len > 2 && splitted[len - 2] == "bun"
            ? "bun"
            : "deno"
          : ext == "go"
@@ -61,11 +62,11 @@ async function dev(opts: GlobalOptions & { filter?: string }) {
          : ext == "ps1"
          ? "powershell"
          : ext == "sql"
-          ? splitted.length > 2 && splitted[splitted.length - 2] == "my"
+          ? len > 2 && splitted[len - 2] == "my"
            ? "mysql"
-            : splitted.length > 2 && splitted[splitted.length - 2] == "bq"
+            : len > 2 && splitted[len - 2] == "bq"
            ? "bigquery"
-            : splitted.length > 2 && splitted[splitted.length - 2] == "sf"
+            : len > 2 && splitted[len - 2] == "sf"
            ? "snowflake"
            : "postgresql"
          : ext == "gql"
--- a/cli/main.ts
+++ b/cli/main.ts
@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
  }
 });

-export const VERSION = "v1.191.0";
+export const VERSION = "v1.202.1";

 let command: any = new Command()
  .name("wmill")
--- a/cli/script.ts
+++ b/cli/script.ts
@@ -151,6 +151,12 @@ export async function handleFile(
          lock: typed?.lock,
          parent_hash: remote.hash,
          schema: typed?.schema,
+          tag: typed?.tag,
+          ws_error_handler_muted: typed?.ws_error_handler_muted,
+          dedicated_worker: typed?.dedicated_worker,
+          cache_ttl: typed?.cache_ttl,
+          concurrency_time_window_s: typed?.concurrency_time_window_s,
+          concurrent_limit: typed?.concurrent_limit,
        },
      });
    } else {
@@ -171,6 +177,12 @@ export async function handleFile(
          lock: typed?.lock,
          parent_hash: undefined,
          schema: typed?.schema,
+          tag: typed?.tag,
+          ws_error_handler_muted: typed?.ws_error_handler_muted,
+          dedicated_worker: typed?.dedicated_worker,
+          cache_ttl: typed?.cache_ttl,
+          concurrency_time_window_s: typed?.concurrency_time_window_s,
+          concurrent_limit: typed?.concurrent_limit,
        },
      });
    }
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,10 +30,7 @@ services:
      - 8000
    environment:
      - DATABASE_URL=${DATABASE_URL}
-      - RUST_LOG=info
      - MODE=server
-      ## You can set the number of workers to 1 and not need any separate worker service but not recommended
-      - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
    depends_on:
      db:
        condition: service_healthy
@@ -50,10 +47,7 @@ services:
    restart: unless-stopped
    environment:
      - DATABASE_URL=${DATABASE_URL}
-      - RUST_LOG=info
      - MODE=worker
-      - KEEP_JOB_DIR=false
-      - METRICS_ADDR=false
      - WORKER_GROUP=default
    depends_on:
      db:
@@ -78,16 +72,15 @@ services:
    restart: unless-stopped
    environment:
      - DATABASE_URL=${DATABASE_URL}
-      - RUST_LOG=info
      - MODE=worker
      - WORKER_GROUP=native
-      - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
    depends_on:
      db:
        condition: service_healthy

  lsp:
    image: ghcr.io/windmill-labs/windmill-lsp:latest
+    pull_policy: always
    restart: unless-stopped
    expose:
      - 3001
@@ -109,13 +102,15 @@ services:
    # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile
+      # - ./certs:/certs # Provide custom certificate files like cert.pem and key.pem to enable HTTPS - See the corresponding section in the Caffyfile
    ports:
      # To change the exposed port, simply change 80:80 to <desired_port>:80. No other changes needed
      - 80:80
-      #- 443:443 # Uncomment to enable HTTPS handling by Caddy
+      # - 443:443 # Uncomment to enable HTTPS handling by Caddy
    environment:
      - BASE_URL=":80"
-      #- BASE_URL=mydomain.com # Uncomment and comment line above to enable HTTPS handling by Caddy
+      # - BASE_URL=":443" # uncomment and comment line above to enable HTTPS via custom certificate and key files
+      # - BASE_URL=mydomain.com # Uncomment and comment line above to enable HTTPS handling by Caddy

 volumes:
  db_data: null
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.191.0",
+	"version": "1.202.1",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.191.0",
+			"version": "1.202.1",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",
@@ -103,7 +103,7 @@
 				"tailwindcss": "^3.3.2",
 				"tslib": "^2.6.1",
 				"typescript": "^5.1.3",
-				"vite": "^4.4.11",
+				"vite": "^4.5.0",
 				"vite-plugin-circular-dependency": "^0.2.1",
 				"vite-plugin-monaco-editor": "^1.1.0",
 				"yootils": "^0.3.1"
@@ -9751,9 +9751,9 @@
 			}
 		},
 		"node_modules/vite": {
-			"version": "4.4.11",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
-			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
+			"version": "4.5.0",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.5.0.tgz",
+			"integrity": "sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==",
 			"dev": true,
 			"dependencies": {
 				"esbuild": "^0.18.10",
@@ -16927,9 +16927,9 @@
 			}
 		},
 		"vite": {
-			"version": "4.4.11",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz",
-			"integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==",
+			"version": "4.5.0",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.5.0.tgz",
+			"integrity": "sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==",
 			"dev": true,
 			"requires": {
 				"esbuild": "^0.18.10",
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.191.0",
+	"version": "1.202.1",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -66,7 +66,7 @@
 		"tailwindcss": "^3.3.2",
 		"tslib": "^2.6.1",
 		"typescript": "^5.1.3",
-		"vite": "^4.4.11",
+		"vite": "^4.5.0",
 		"vite-plugin-circular-dependency": "^0.2.1",
 		"vite-plugin-monaco-editor": "^1.1.0",
 		"yootils": "^0.3.1"
--- a/frontend/src/lib/assets/app.css
+++ b/frontend/src/lib/assets/app.css
@@ -29,6 +29,10 @@
 		display: none !important;
 	}

+	.monaco-editor .suggest-widget {
+		z-index: 10001 !important;
+	}
+
 	.nowrap pre code.hljs {
 		whitespace: normal !important;
 	}
--- a/frontend/src/lib/cloud.ts
+++ b/frontend/src/lib/cloud.ts
@@ -1,27 +1,5 @@
 import { BROWSER } from 'esm-env'
-import { premiumStore, userStore, workspaceStore } from './stores'
-import { getUserExt } from './user'
-import { WorkspaceService } from './gen'

 export function isCloudHosted(): boolean {
 	return BROWSER && window.location.hostname == 'app.windmill.dev'
 }
-
-if (BROWSER) {
-	workspaceStore.subscribe(async (workspace) => {
-		if (workspace) {
-			try {
-				localStorage.setItem('workspace', String(workspace))
-			} catch (e) {
-				console.error('Could not persist workspace to local storage', e)
-			}
-			const user = await getUserExt(workspace)
-			userStore.set(user)
-			if (isCloudHosted() && user?.is_admin) {
-				premiumStore.set(await WorkspaceService.getPremiumInfo({ workspace }))
-			}
-		} else {
-			userStore.set(undefined)
-		}
-	})
-}
--- a/Show More
+++ b/Show More