Result node message

* critical alert ui

* updating ui, backend logic

* revert

* type check fix npm

* checking out cli files from main

* moving alert icon

* adding sqlx mock data

* more sqlx changes

* feat(frontend): nodes from flow can be connected directly in expr input through a plug icon (#4652)

* Add flow prop picker

# Conflicts:
#	frontend/src/lib/components/propertyPicker/PropPicker.svelte

* fix unwanted copy

* cleaning

* Fix unset context

* move button and always display input

* fix unwanted proppicker display

* update

* update

* clean all

* clean all

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>

* replace hide/show with toggle

* adding mutable setting and navigation to settings to configure channels

* merge fix

* ee non ee changees

* auto-acknowledge when muted

* pr comments

* fix bad log

* user inner modal component

* update unaknowledge alerts after acknowledging from modal

* aknowledge -> acknowledge

* format

* adding backend support for workspace critical alerts

* immediately check for alerts

* immediately check for alerts

* adding openapi changes

* simplify loading of superadmin/ee

* update modal logic

* frontend logic update

* sqlx prepare

* adding mute functionality for workspace critical alerts + show alerts button for instance and workspace settings

* npm / rust warnings & errors

* reverting non-mac cargo toml

* reverting non-mac cargo toml

* mute toggles in critical alert ui

* ui polish

* adding acknowledged_workspace column

* sqlx prepare

* make sure we wait for all stores to be loaded

* adding workspace mute logic to report_critical alert

* auto ack recovered alerts

* toggle workspace as superadmin and critical alerts in logs menu

* adding critical alert button if width < 786px

* refresh on change

* Move notification to logs (#4698)

* Move notification to logs

* removing unused button

---------

Co-authored-by: Alexander Petric <petric.al@gmail.com>

* tailwind typo

* auto ack on mute

* don't change deref

* sqlx prep

* avoid renaming db column, keep acknowledged instead of acknowledged_global

* hide critical alert menu when not ee

* if workspace muted, also acknowledge global when workspace set

* removing save button for mute setting and improve ux/responsiveness

* fix: deployment callbacks have a concurrency limit of 1 on same path

* sqlx prep

* ee-repo ref

* z-[9999] for modal

---------

Co-authored-by: Guilhem <guilhemlemouel@gmail.com>
Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/.sqlx/query-00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "00588a40dde5189ac1c61505f17acb0f4c244c60477427505bf5bd1b104d3bf9"
+}

backend/.sqlx/query-044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n        VALUES ('critical_error', $1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "044e2b428ee6e2dd4543c87ad8835e239cf7567d18b8b3fa6608ea3a9d206ca7"
+}

backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM healthchecks WHERE check_type = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a"
-}

backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json

@@ -117,6 +117,11 @@
         "ordinal": 22,
         "name": "deploy_ui",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 23,
+        "name": "mute_critical_alerts",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -147,6 +152,7 @@
       true,
       false,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)\n        VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Bool",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "28987898c7e4b172d466bf08f33c2da733e71f8a253b0122c17b9e021919809e"
+}

backend/.sqlx/query-344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177.json

@@ -0,0 +1,54 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n                 FROM alerts\n                 WHERE COALESCE(acknowledged, false) = $1\n                 ORDER BY created_at DESC\n                 LIMIT $2 OFFSET $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      null,
+      true
+    ]
+  },
+  "hash": "344b3a5d9683273a956b5156fed5ab9fdff1a7252198e4ae290d8c8f937ff177"
+}

backend/.sqlx/query-3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "3a94ad52c6b7cde844fa868167248cd9ff63e5fdfa1d93d8fbec32a257b6b05e"
-}

backend/.sqlx/query-4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4a3a8207627418ba7b7eabaccd7ec72884f2a1eacf0ab0e0c6ccbf56c654c14a"
+}

backend/.sqlx/query-4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4d22084a5d9860832f30e8f08cbfa1848ed3c1336fa4790f45b8189c4ac97d91"
-}

backend/.sqlx/query-4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             WHERE acknowledged = $1\n             ORDER BY created_at DESC \n             LIMIT $2 OFFSET $3",
+  "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id\n                 FROM alerts\n                 ORDER BY created_at DESC\n                 LIMIT $1 OFFSET $2",
   "describe": {
     "columns": [
       {
@@ -27,11 +27,15 @@
         "ordinal": 4,
         "name": "acknowledged",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Text"
       }
     ],
     "parameters": {
       "Left": [
-        "Bool",
         "Int8",
         "Int8"
       ]
@@ -41,8 +45,9 @@
       false,
       false,
       false,
+      null,
       true
     ]
   },
-  "hash": "0b955f2cff82a2d4ba3840588143e08952f029480d4a42503ecc3c5e70437995"
+  "hash": "4d30c5a2894d655741d167f5589f5816583f0bae78a0dd3270dedb377dfa944a"
 }

backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json

@@ -117,6 +117,11 @@
         "ordinal": 22,
         "name": "deploy_ui",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 23,
+        "name": "mute_critical_alerts",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -147,6 +152,7 @@
       true,
       false,
       true,
+      true,
       true
     ]
   },

backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts\n         SET\n           acknowledged = true,\n           acknowledged_workspace = CASE\n             WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n             ELSE acknowledged_workspace\n           END\n         WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a"
+}

backend/.sqlx/query-777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT id, alert_type, message, created_at, acknowledged \n             FROM alerts \n             ORDER BY created_at DESC \n             LIMIT $1 OFFSET $2",
+  "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n                 FROM alerts\n                 WHERE workspace_id = $1\n                 ORDER BY created_at DESC\n                 LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -27,10 +27,16 @@
         "ordinal": 4,
         "name": "acknowledged",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Text"
       }
     ],
     "parameters": {
       "Left": [
+        "Text",
         "Int8",
         "Int8"
       ]
@@ -40,8 +46,9 @@
       false,
       false,
       false,
+      null,
       true
     ]
   },
-  "hash": "cc5ab80241b88c5befea279f16c4ec68cec17b31dcd277b321f652917346496b"
+  "hash": "777e7084edf9a5d14f299ff479ae43c8ead47a6a531d4b031a1342c9b2f16506"
 }

backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts \n         SET\n           acknowledged = true,\n           acknowledged_workspace = CASE\n             WHEN $1::text IS NOT NULL THEN true\n             ELSE acknowledged_workspace\n           END\n         WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n            OR ($1::text IS NULL)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3"
+}

backend/.sqlx/query-7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d.json

@@ -0,0 +1,55 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id\n                 FROM alerts\n                 WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2\n                 ORDER BY created_at DESC\n                 LIMIT $3 OFFSET $4",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "alert_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "acknowledged",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Bool",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      null,
+      true
+    ]
+  },
+  "hash": "7c5a29de07cbe42326a15d4d7fd9714c20b767508e27368be6de38d7b18a3a4d"
+}

backend/.sqlx/query-a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE alerts SET acknowledged = true",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "a59fae29ebcc9aa53308b777ead3d2b652618ac454f139db738f4499cb4eb079"
+}

backend/.sqlx/query-a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE alerts SET acknowledged = true WHERE acknowledged = false",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "a7c5008aa7ea43d0afac7d9f19846976ed7af2e90270902f001115e023cb947d"
-}

backend/.sqlx/query-be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE alerts SET acknowledged = true WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "be3ae231557e794172336bc27d725f862dcf039bbb3d75ced9d54c86f53d2580"
-}

backend/.sqlx/query-e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "mute_critical_alerts",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "e05dbe046e846c092a96b6b0a9d872c721169d418586be2d15cee6b929049098"
+}

backend/.sqlx/query-f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Bool",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f22168826350797e88153b65a5b1e906a7868f34c062f154b2ee4f24c57aa5c3"
+}

backend/ee-repo-ref.txt

@@ -1 +1 @@
-177895136549cdb7c2f2a26ba3b654d76e60d845
+51dcbf93b0d127af9f33fa346cc63fcd2475d4fa

backend/migrations/20241108201445_alert_add_workspace_id.down.sql

@@ -0,0 +1,4 @@
+ALTER TABLE alerts DROP COLUMN workspace_id;
+ALTER TABLE alerts DROP COLUMN acknowledged_workspace;
+ALTER TABLE alerts DROP COLUMN resource;
+ALTER TABLE workspace_settings DROP COLUMN mute_critical_alerts;

backend/migrations/20241108201445_alert_add_workspace_id.up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE alerts ADD COLUMN workspace_id TEXT DEFAULT NULL;
+ALTER TABLE alerts ADD COLUMN acknowledged_workspace BOOL DEFAULT NULL;
+ALTER TABLE alerts ADD COLUMN resource TEXT DEFAULT NULL;
+ALTER TABLE workspace_settings ADD COLUMN mute_critical_alerts BOOL DEFAULT NULL;

backend/src/main.rs

@@ -30,14 +30,15 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE
 
 use windmill_common::{
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
-        CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
-        ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
-        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
-        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
+        DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
+        EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING,
+        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
+        REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
+        RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
+        TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -66,18 +67,19 @@ use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
-    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
+    PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
 };
 
 use crate::monitor::{
     initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
     load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, monitor_pool,
     reload_base_url_setting, reload_bunfig_install_scopes_setting,
-    reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
-    reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
-    reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
+    reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting,
+    reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
+    reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
+    reload_npm_config_registry_setting, reload_pip_index_url_setting,
     reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
-    reload_worker_config, reload_critical_alert_mute_ui_setting,
+    reload_worker_config,
 };
 
 #[cfg(feature = "parquet")]
@@ -995,13 +997,14 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
     for x in [
         LOCK_CACHE_DIR,
         TMP_LOGS_DIR,
-        PIP_CACHE_DIR,
         UV_CACHE_DIR,
         TAR_PIP_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
         BUN_CACHE_DIR,
+        PY311_CACHE_DIR,
+        PIP_CACHE_DIR,
         BUN_DEPSTAR_CACHE_DIR,
         BUN_BUNDLE_CACHE_DIR,
         GO_CACHE_DIR,

backend/src/monitor.rs

@@ -34,12 +34,12 @@ use windmill_common::{
     error,
     flow_status::FlowStatusModule,
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
-        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
-        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
-        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
-        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
+        DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
+        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -54,8 +54,8 @@ use windmill_common::{
         update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG,
         WORKER_CONFIG, WORKER_GROUP,
     },
-    BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
-    METRICS_DEBUG_ENABLED, METRICS_ENABLED, CRITICAL_ALERT_MUTE_UI_ENABLED
+    BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
+    HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
 };
 use windmill_queue::cancel_job;
 use windmill_worker::{
@@ -231,13 +231,20 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
 }
 
 pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
-    let mute = load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await;
-    match mute {
-        Ok(Some(serde_json::Value::Bool(t))) => {
-            CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
+    if let Ok(Some(serde_json::Value::Bool(t))) =
+        load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await
+    {
+        CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
+
+        if t {
+            if let Err(e) = sqlx::query!("UPDATE alerts SET acknowledged = true")
+                .execute(db)
+                .await
+            {
+                tracing::error!("Error updating alerts: {}", e.to_string());
+            }
         }
-        _ => (),
-    };
+    }
     Ok(())
 }
 
@@ -1323,7 +1330,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                 ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id)
                 .execute(db).await;
             tracing::error!(error_message);
-            report_critical_error(error_message, db.clone()).await;
+            report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await;
         }
     }
 
@@ -1437,7 +1444,7 @@ async fn handle_zombie_flows(
                 flow.id, flow.workspace_id
             );
             tracing::error!(error_message);
-            report_critical_error(error_message, db.clone()).await;
+            report_critical_error(error_message, db.clone(), Some(&flow.workspace_id), None).await;
             // if the flow hasn't started and is a zombie, we can simply restart it
             sqlx::query!(
                 "UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false",
@@ -1457,7 +1464,7 @@ async fn handle_zombie_flows(
                     format!("Flow {id} was cancelled because it")
                 }
             );
-            report_critical_error(reason.clone(), db.clone()).await;
+            report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
             cancel_zombie_flow_job(db, flow, &rsmq, reason).await?;
         }
     }

backend/windmill-api/openapi.yaml

@@ -1701,6 +1701,8 @@ paths:
                     type: string
                   default_scripts:
                     $ref: "#/components/schemas/WorkspaceDefaultScripts"
+                  mute_critical_alerts:
+                    type: boolean
                 required:
                   - code_completion_enabled
                   - automatic_billing
@@ -2717,6 +2719,112 @@ paths:
                 items:
                   $ref: "#/components/schemas/ContextualVariable"
 
+  /w/{workspace}/workspaces/critical_alerts:
+    get:
+      summary: Get all critical alerts for this workspace
+      operationId: workspaceGetCriticalAlerts
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - in: query
+          name: page
+          schema:
+            type: integer
+            default: 1
+            description: The page number to retrieve (minimum value is 1)
+        - in: query
+          name: page_size
+          schema:
+            type: integer
+            default: 10
+            maximum: 100
+            description: Number of alerts per page (maximum is 100)
+        - in: query
+          name: acknowledged
+          schema:
+            type: boolean
+            nullable: true
+            description: Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts
+      responses:
+        "200":
+          description: Successfully retrieved all critical alerts
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/CriticalAlert'
+
+  /w/{workspace}/workspaces/critical_alerts/{id}/acknowledge:
+    post:
+      summary: Acknowledge a critical alert for this workspace
+      operationId: workspaceAcknowledgeCriticalAlert
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: integer
+          description: The ID of the critical alert to acknowledge
+      responses:
+        "200":
+          description: Successfully acknowledged the critical alert
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Critical alert acknowledged"
+
+  /w/{workspace}/workspaces/critical_alerts/acknowledge_all:
+    post:
+      summary: Acknowledge all unacknowledged critical alerts for this workspace
+      operationId: workspaceAcknowledgeAllCriticalAlerts
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: Successfully acknowledged all unacknowledged critical alerts.
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "All unacknowledged critical alerts acknowledged"
+
+  /w/{workspace}/workspaces/critical_alerts/mute:
+    post:
+      summary: Mute critical alert UI for this workspace
+      operationId: workspaceMuteCriticalAlertsUI
+      tags:
+        - setting
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: Boolean flag to mute critical alerts.
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                mute_critical_alerts:
+                  type: boolean
+                  description: Whether critical alerts should be muted.
+                  example: true
+      responses:
+        '200':
+          description: Successfully updated mute critical alert settings.
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Updated mute critical alert UI settings for workspace: workspace_id"
+
   /oauth/login_callback/{client_name}:
     post:
       security: []
@@ -12945,3 +13053,7 @@ components:
           type: boolean
           nullable: true
           description: Acknowledgment status of the alert, can be true, false, or null if not set
+        workspace_id:
+          type: string
+          nullable: true
+          description: Workspace id if the alert is in the scope of a workspace

backend/windmill-api/src/resources.rs

@@ -300,7 +300,7 @@ async fn get_resource(
     .await?;
     tx.commit().await?;
     if resource_o.is_none() {
-        explain_resource_perm_error(&path, &w_id, &db).await?;
+        explain_resource_perm_error(&path, &w_id, &db, &authed).await?;
     }
     let resource = not_found_if_none(resource_o, "Resource", path)?;
     Ok(Json(resource))
@@ -343,7 +343,7 @@ async fn get_resource_value(
 
     tx.commit().await?;
     if value_o.is_none() {
-        explain_resource_perm_error(&path, &w_id, &db).await?;
+        explain_resource_perm_error(&path, &w_id, &db, &authed).await?;
     }
 
     let value = not_found_if_none(value_o, "Resource", path)?;
@@ -354,6 +354,7 @@ async fn explain_resource_perm_error(
     path: &str,
     w_id: &str,
     db: &sqlx::Pool<Postgres>,
+    authed: &ApiAuthed,
 ) -> windmill_common::error::Result<()> {
     let extra_perms = sqlx::query_scalar!(
         "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2",
@@ -378,12 +379,12 @@ async fn explain_resource_perm_error(
         .fetch_optional(db)
         .await?;
         return Err(Error::NotAuthorized(format!(
-            "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}",
+            "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}",
             serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default()
         )));
     } else {
         return Err(Error::NotAuthorized(format!(
-            "Resource exists but you don't have access to it:\nresource perms: {}",
+            "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}",
             serde_json::to_string_pretty(&extra_perms).unwrap_or_default()
         )));
     }
@@ -457,7 +458,7 @@ pub async fn get_resource_value_interpolated_internal(
     .await?;
     tx.commit().await?;
     if value_o.is_none() {
-        explain_resource_perm_error(path, workspace, db).await?;
+        explain_resource_perm_error(path, workspace, db, &authed).await?;
     }
 
     let value = not_found_if_none(value_o, "Resource", path)?;

backend/windmill-api/src/settings.rs

@@ -434,67 +434,15 @@ pub async fn test_critical_channels() -> Result<String> {
     Ok("Critical channels require EE".to_string())
 }
 
-use serde::Serialize;
-
-#[derive(Serialize)]
-pub struct CriticalAlert {
-    id: i32,
-    alert_type: String,
-    message: String,
-    created_at: chrono::DateTime<chrono::Utc>,
-    acknowledged: Option<bool>,
-}
-
-#[cfg(feature = "enterprise")]
-#[derive(Deserialize)]
-pub struct AlertQueryParams {
-    pub page: Option<i32>,
-    pub page_size: Option<i32>,
-    pub acknowledged: Option<bool>,
-}
-
 #[cfg(feature = "enterprise")]
 pub async fn get_critical_alerts(
     Extension(db): Extension<DB>,
     authed: ApiAuthed,
-    Query(params): Query<AlertQueryParams>,
-) -> JsonResult<Vec<CriticalAlert>> {
+    Query(params): Query<crate::utils::AlertQueryParams>,
+) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
     require_super_admin(&db, &authed.email).await?;
 
-    // Default pagination values if not provided
-    let page = params.page.unwrap_or(1).max(1);
-    let page_size = params.page_size.unwrap_or(10).min(100) as i64;
-    let offset = ((page - 1) * page_size as i32) as i64;
-
-    let alerts = if let Some(acknowledged) = params.acknowledged {
-        sqlx::query_as!(
-            CriticalAlert,
-            "SELECT id, alert_type, message, created_at, acknowledged 
-             FROM alerts 
-             WHERE acknowledged = $1
-             ORDER BY created_at DESC 
-             LIMIT $2 OFFSET $3",
-            acknowledged,
-            page_size,
-            offset
-        )
-        .fetch_all(&db)
-        .await?
-    } else {
-        sqlx::query_as!(
-            CriticalAlert,
-            "SELECT id, alert_type, message, created_at, acknowledged 
-             FROM alerts 
-             ORDER BY created_at DESC 
-             LIMIT $1 OFFSET $2",
-            page_size,
-            offset
-        )
-        .fetch_all(&db)
-        .await?
-    };
-
-    Ok(Json(alerts))
+    crate::utils::get_critical_alerts(db, params, None).await
 }
 
 #[cfg(not(feature = "enterprise"))]
@@ -510,15 +458,7 @@ pub async fn acknowledge_critical_alert(
 ) -> error::Result<String> {
     require_super_admin(&db, &authed.email).await?;
     
-    sqlx::query!(
-        "UPDATE alerts SET acknowledged = true WHERE id = $1",
-        id
-    )
-    .execute(&db)
-    .await?;
-    
-    tracing::info!("Acknowledged critical alert with id: {}", id);
-    Ok("Critical alert acknowledged".to_string())
+    crate::utils::acknowledge_critical_alert(db, None, id).await
 }
 
 #[cfg(not(feature = "enterprise"))]
@@ -533,14 +473,7 @@ pub async fn acknowledge_all_critical_alerts(
 ) -> error::Result<String> {
     require_super_admin(&db, &authed.email).await?;
 
-    sqlx::query!(
-        "UPDATE alerts SET acknowledged = true WHERE acknowledged = false"
-    )
-    .execute(&db)
-    .await?;
-
-    tracing::info!("Acknowledged all unacknowledged critical alerts");
-    Ok("All unacknowledged critical alerts acknowledged".to_string())
+    crate::utils::acknowledge_all_critical_alerts(db, None).await
 }
 
 #[cfg(not(feature = "enterprise"))]

backend/windmill-api/src/utils.rs

@@ -16,6 +16,12 @@ use windmill_common::{
     DB,
 };
 
+#[cfg(feature = "enterprise")]
+use windmill_common::error::JsonResult;
+
+#[cfg(feature = "enterprise")]
+use axum::Json;
+
 #[derive(Deserialize)]
 pub struct WithStarredInfoQuery {
     pub with_starred_info: Option<bool>,
@@ -162,3 +168,159 @@ pub fn content_plain(body: Body) -> Response {
         .body(body)
         .unwrap()
 }
+
+use serde::Serialize;
+
+#[derive(Serialize)]
+pub struct CriticalAlert {
+    id: i32,
+    alert_type: String,
+    message: String,
+    created_at: chrono::DateTime<chrono::Utc>,
+    acknowledged: Option<bool>,
+    workspace_id: Option<String>,
+}
+
+#[cfg(feature = "enterprise")]
+#[derive(Deserialize, Debug)]
+pub struct AlertQueryParams {
+    pub page: Option<i32>,
+    pub page_size: Option<i32>,
+    pub acknowledged: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn get_critical_alerts(
+    db: DB,
+    params: AlertQueryParams,
+    workspace_id: Option<String>,
+) -> JsonResult<Vec<CriticalAlert>> {
+    let page = params.page.unwrap_or(1).max(1);
+    let page_size = params.page_size.unwrap_or(10).min(100) as i64;
+    let offset = ((page - 1) * page_size as i32) as i64;
+
+    let alerts = if let Some(workspace_id) = workspace_id {
+        // `workspace_id` is provided => workspace admin
+        if params.acknowledged.is_none() {
+            // Case: return all rows where `workspace_id` matches
+            sqlx::query_as!(
+                CriticalAlert,
+                "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id
+                 FROM alerts
+                 WHERE workspace_id = $1
+                 ORDER BY created_at DESC
+                 LIMIT $2 OFFSET $3",
+                workspace_id,
+                page_size,
+                offset
+            )
+            .fetch_all(&db)
+            .await?
+        } else {
+            // Case: return rows where `acknowledged_workspace` matches `params.acknowledged`
+            sqlx::query_as!(
+                CriticalAlert,
+                "SELECT id, alert_type, message, created_at, COALESCE(acknowledged_workspace, false) AS acknowledged, workspace_id
+                 FROM alerts
+                 WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2
+                 ORDER BY created_at DESC
+                 LIMIT $3 OFFSET $4",
+                workspace_id,
+                params.acknowledged,
+                page_size,
+                offset
+            )
+            .fetch_all(&db)
+            .await?
+        }
+    } else {
+        // `workspace_id` is not provided => superadmin
+        if params.acknowledged.is_none() {
+            // Case: Return all rows unfiltered with global acknowledged as acknowledged
+            sqlx::query_as!(
+                CriticalAlert,
+                "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id
+                 FROM alerts
+                 ORDER BY created_at DESC
+                 LIMIT $1 OFFSET $2",
+                page_size,
+                offset
+            )
+            .fetch_all(&db)
+            .await?
+        } else {
+            // Case: Return rows where global acknowledged matches params.acknowledged
+            sqlx::query_as!(
+                CriticalAlert,
+                "SELECT id, alert_type, message, created_at, COALESCE(acknowledged, false) AS acknowledged, workspace_id
+                 FROM alerts
+                 WHERE COALESCE(acknowledged, false) = $1
+                 ORDER BY created_at DESC
+                 LIMIT $2 OFFSET $3",
+                params.acknowledged,
+                page_size,
+                offset
+            )
+            .fetch_all(&db)
+            .await?
+        }
+    };
+
+    Ok(Json(alerts))
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_critical_alert(
+    db: DB,
+    workspace_id: Option<String>,
+    id: i32,
+) -> error::Result<String> {
+    sqlx::query!(
+        "UPDATE alerts
+         SET
+           acknowledged = true,
+           acknowledged_workspace = CASE
+             WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true
+             ELSE acknowledged_workspace
+           END
+         WHERE id = $1",
+        id,
+        workspace_id
+    )
+    .execute(&db)
+    .await?;
+
+    tracing::info!(
+        "Acknowledged critical alert with id: {}{}",
+        id,
+        workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w))
+    );
+    Ok("Critical alert acknowledged".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_all_critical_alerts(
+    db: DB,
+    workspace_id: Option<String>,
+) -> error::Result<String> {
+    sqlx::query!(
+        "UPDATE alerts 
+         SET
+           acknowledged = true,
+           acknowledged_workspace = CASE
+             WHEN $1::text IS NOT NULL THEN true
+             ELSE acknowledged_workspace
+           END
+         WHERE ($1::text IS NOT NULL AND workspace_id = $1)
+            OR ($1::text IS NULL)",
+        workspace_id
+    )
+    .execute(&db)
+    .await?;
+
+    tracing::info!(
+        "Acknowledged all unacknowledged critical alerts{}",
+        workspace_id.map_or_else(|| "".to_string(), |w| format!(" for workspace_id: {}", w))
+    );
+    Ok("All unacknowledged critical alerts acknowledged".to_string())
+}

backend/windmill-api/src/websocket_triggers.rs

@@ -754,12 +754,14 @@ async fn disable_with_error(db: &DB, ws_trigger: &WebsocketTrigger, error: Strin
     )
     .execute(db).await {
         Ok(_) => {
-            report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone()).await;
+            report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone(), Some(&ws_trigger.workspace_id), None).await;
         },
         Err(disable_err) => {
             report_critical_error(
                 format!("Could not disable websocket {} with err {}, disabling because of error {}", ws_trigger.path, disable_err, error), 
-                db.clone()
+                db.clone(),
+                Some(&ws_trigger.workspace_id),
+                None,
             ).await;
         }
     }
@@ -896,7 +898,7 @@ async fn listen_to_websocket(
                                                                 }
                                                                 if should_handle {
                                                                     if let Err(err) = run_job(&db, rsmq.clone(), &ws_trigger, text).await {
-                                                                        report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone()).await;
+                                                                        report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone(), Some(&ws_trigger.workspace_id), None).await;
                                                                     };
                                                                 }
                                                             },

backend/windmill-api/src/workspaces.rs

@@ -119,7 +119,11 @@ pub fn workspaced_service() -> Router {
         .route("/change_workspace_name", post(change_workspace_name))
         .route("/change_workspace_id", post(change_workspace_id))
         .route("/usage", get(get_usage))
-        .route("/used_triggers", get(get_used_triggers));
+        .route("/used_triggers", get(get_used_triggers))
+        .route("/critical_alerts", get(get_critical_alerts))
+        .route("/critical_alerts/:id/acknowledge", post(acknowledge_critical_alert))
+        .route("/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts))
+        .route("/critical_alerts/mute", post(mute_critical_alerts));
 
     #[cfg(feature = "stripe")]
     {
@@ -180,6 +184,7 @@ pub struct WorkspaceSettings {
     pub default_app: Option<String>,
     pub automatic_billing: bool,
     pub default_scripts: Option<serde_json::Value>,
+    pub mute_critical_alerts: Option<bool>,
 }
 
 #[derive(FromRow, Serialize, Debug)]
@@ -3075,3 +3080,93 @@ async fn get_usage(Extension(db): Extension<DB>, Path(w_id): Path<String>) -> Re
     .unwrap_or(0);
     Ok(usage.to_string())
 }
+
+#[cfg(feature = "enterprise")]
+pub async fn get_critical_alerts(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    authed: ApiAuthed,
+    Query(params): Query<crate::utils::AlertQueryParams>,
+) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    crate::utils::get_critical_alerts(db, params, Some(w_id)).await
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn get_critical_alerts() -> Error {
+    Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_critical_alert(
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, i32)>,
+    authed: ApiAuthed,
+) -> Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await 
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_critical_alert() -> Error {
+    Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn acknowledge_all_critical_alerts(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    authed: ApiAuthed,
+) -> Result<String> {
+    require_admin(authed.is_admin, &authed.username)?;
+    crate::utils::acknowledge_all_critical_alerts(db, Some(w_id)).await
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn acknowledge_all_critical_alerts() -> Error {
+    Error::NotFound("Critical Alerts require EE".to_string())
+}
+
+
+#[cfg(feature = "enterprise")]
+#[derive(Deserialize)]
+pub struct MuteCriticalAlertRequest {
+    pub mute_critical_alerts: Option<bool>,
+}
+
+#[cfg(feature = "enterprise")]
+async fn mute_critical_alerts(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
+    Json(m_r): Json<MuteCriticalAlertRequest>,
+) -> Result<String> {
+    require_admin(is_admin, &username)?;
+
+    let mute_alerts = m_r.mute_critical_alerts.unwrap_or(false);
+
+    if mute_alerts {
+        sqlx::query!(
+            "UPDATE alerts SET acknowledged_workspace = true, acknowledged = true WHERE workspace_id = $1",
+            &w_id
+        )
+    .execute(&db)
+    .await?;
+    }
+
+    sqlx::query!(
+        "UPDATE workspace_settings SET mute_critical_alerts = $1 WHERE workspace_id = $2",
+        mute_alerts,
+        &w_id
+    )
+    .execute(&db)
+    .await?;
+
+    Ok(format!("Updated mute criticital alert ui settings for workspace: {}", &w_id))
+}
+
+#[cfg(not(feature = "enterprise"))]
+pub async fn mute_critical_alerts() -> Error {
+    Error::NotFound("Critical Alerts require EE".to_string())
+}

backend/windmill-common/src/utils.rs

@@ -29,8 +29,11 @@ pub const DEFAULT_PER_PAGE: usize = 1000;
 pub const GIT_VERSION: &str =
     git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 
-use std::sync::atomic::Ordering;
 use crate::CRITICAL_ALERT_MUTE_UI_ENABLED;
+use std::sync::atomic::Ordering;
+
+#[cfg(feature = "enterprise")]
+use crate::worker::CLOUD_HOSTED;
 
 lazy_static::lazy_static! {
     pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
@@ -249,15 +252,40 @@ pub fn generate_lock_id(database_name: &str) -> i64 {
     0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64)
 }
 
-pub async fn report_critical_error(error_message: String, _db: DB) -> () {
+pub async fn report_critical_error(
+    error_message: String,
+    _db: DB,
+    workspace_id: Option<&str>,
+    resource: Option<&str>,
+) -> () {
     tracing::error!("CRITICAL ERROR: {error_message}");
 
-    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+    let mute_global = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
+    let mute_workspace = if let Some(workspace_id) = workspace_id {
+        match fetch_mute_workspace(&_db, workspace_id).await {
+            Ok(flag) => flag,
+            Err(err) => {
+                tracing::error!("Error fetching mute_workspace: {}", err);
+                false
+            }
+        }
+    } else {
+        false
+    };
+
+    // we ack_global if mute_global is true, or if mute_workspace is true
+    // but we ignore global mute setting for ack_workspace
+    let acknowledge_workspace = mute_workspace;
+    let acknowledge_global = mute_global || mute_workspace;
 
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('critical_error', $1, $2)",
+        "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)
+        VALUES ('critical_error', $1, $2, $3, $4, $5)",
         error_message,
-        mute
+        acknowledge_global,
+        acknowledge_workspace,
+        workspace_id,
+        resource,
     )
     .execute(&_db)
     .await
@@ -266,30 +294,86 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
     }
 
     #[cfg(feature = "enterprise")]
-    send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
+    if *CLOUD_HOSTED && workspace_id.is_some() {
+        tracing::error!(error_message)
+    } else {
+        send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
+    }
 }
 
-pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
+pub async fn report_recovered_critical_error(
+    message: String,
+    _db: DB,
+    workspace_id: Option<&str>,
+    resource: Option<&str>,
+) -> () {
     tracing::info!("RECOVERED CRITICAL ERROR: {message}");
 
-    let mute = CRITICAL_ALERT_MUTE_UI_ENABLED.load(Ordering::Relaxed);
-
     if let Err(err) = sqlx::query!(
-        "INSERT INTO alerts (alert_type, message, acknowledged) VALUES ('recovered_critical_error', $1, $2)",
+        "INSERT INTO alerts (alert_type, message, acknowledged, acknowledged_workspace, workspace_id, resource)
+        VALUES ('recovered_critical_error', $1, $2, $3, $4, $5)",
         message,
-        mute
+        true,
+        true,
+        workspace_id,
+        resource,
     )
     .execute(&_db)
     .await
     {
-        tracing::error!("Failed to save critical error to database: {}", err);
+        tracing::error!("Failed to save recovered critical error to database: {}", err);
     }
+
+    // acknowledge all alerts with the same resource
+    if let Some(resource) = resource {
+        if let Err(err) = sqlx::query!(
+            "UPDATE alerts SET acknowledged = true, acknowledged_workspace = true WHERE resource = $1 AND alert_type = 'critical_error'",
+            resource,
+        )
+        .execute(&_db)
+        .await
+        {
+            tracing::error!("Failed to acknowledge critical error alerts for resource {}: {}", resource, err);
+        }
+    }
+
     #[cfg(feature = "enterprise")]
-    send_critical_alert(
-        message,
-        &_db,
-        CriticalAlertKind::RecoveredCriticalError,
-        None,
-    )
-    .await;
+    if *CLOUD_HOSTED && workspace_id.is_some() {
+        tracing::error!(message);
+    } else {
+        send_critical_alert(
+            message,
+            &_db,
+            CriticalAlertKind::RecoveredCriticalError,
+            None,
+        )
+        .await;
+    }
+}
+
+pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result<bool> {
+    match sqlx::query!(
+        "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1",
+        workspace_id
+    )
+    .fetch_optional(_db)
+    .await
+    {
+        Ok(Some(record)) => Ok(record.mute_critical_alerts.unwrap_or(false)),
+        Ok(None) => {
+            tracing::warn!(
+                "Workspace ID {} not found in workspace_settings table",
+                workspace_id
+            );
+            Ok(false)
+        }
+        Err(err) => {
+            tracing::error!(
+                "Error querying workspace_settings for workspace_id {}: {}",
+                workspace_id,
+                err
+            );
+            Err(Error::SqlErr(err))
+        }
+    }
 }

backend/windmill-common/src/worker.rs

@@ -310,6 +310,8 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
 pub struct PythonAnnotations {
     pub no_cache: bool,
     pub no_uv: bool,
+    pub no_uv_install: bool,
+    pub no_uv_compile: bool,
 }
 
 #[annotations("//")]

backend/windmill-queue/src/jobs.rs

@@ -924,6 +924,8 @@ pub async fn add_completed_job<
                         .unwrap_or("".to_string()),
                 ),
                 db.clone(),
+                Some(&w_id),
+                None,
             )
             .await;
         } else if queued_job.email == SCHEDULE_ERROR_HANDLER_USER_EMAIL {
@@ -992,7 +994,7 @@ pub async fn add_completed_job<
                             "Could not push workspace error handler for failed job ({base_url}/run/{}?workspace={w_id}): {}",
                             queued_job.id,
                             err
-                        ), db.clone())
+                        ), db.clone(), Some(&w_id), None)
                         .await;
                     }
                 }
@@ -1175,10 +1177,10 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel<
                 queued_job.id,
                 err
             );
-            report_critical_error(error_message, db.clone()).await;
+            report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
         }
     } else {
-        report_critical_error(error_message, db.clone()).await;
+        report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
     }
 }
 

backend/windmill-worker/nsjail/download.py.pip.config.proto

@@ -0,0 +1,93 @@
+name: "python download pip"
+
+mode: ONCE
+hostname: "python"
+log_level: ERROR
+time_limit: 900
+
+rlimit_as: 2048
+rlimit_cpu: 1000
+rlimit_fsize: 1024
+rlimit_nofile: 64
+
+envar: "HOME=/user"
+envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
+
+cwd: "/tmp"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+keep_caps: true
+keep_env: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+mount {
+	src: "/dev/null"
+	dst: "/dev/null"
+	is_bind: true
+	rw: true
+}
+
+mount {
+	dst: "/tmp"
+	fstype: "tmpfs"
+	rw: true
+    options: "size=500000000"
+}
+
+
+mount {
+    src: "{WORKER_DIR}/download_deps.py.pip.sh"
+    dst: "/download_deps.sh"
+    is_bind: true
+}
+
+mount {
+    src: "{CACHE_DIR}"
+    dst: "{CACHE_DIR}"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
+exec_bin {
+ 	path: "/bin/sh"
+ 	arg: "/download_deps.sh"
+}
+

backend/windmill-worker/nsjail/download_deps.py.pip.sh

@@ -0,0 +1,24 @@
+#/bin/sh
+
+INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" )
+EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" )
+TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] &&  echo "" || echo "--trusted-host $TRUSTED_HOST")
+
+if [ ! -z "$INDEX_URL" ]
+then
+      echo "\$INDEX_URL is set to $INDEX_URL"
+fi
+
+if [ ! -z "$EXTRA_INDEX_URL" ]
+then
+      echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL"
+fi
+
+if [ ! -z "$TRUSTED_HOST" ]
+then
+      echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
+fi
+
+CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
+echo $CMD
+eval $CMD

backend/windmill-worker/nsjail/download_deps.py.sh

@@ -19,6 +19,18 @@ then
       echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
 fi
 
-CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
+CMD="/usr/local/bin/uv pip install 
+\"$REQ\" 
+--target \"$TARGET\" 
+--no-cache 
+--no-config
+--no-color 
+--no-deps 
+--link-mode=copy
+$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
+--index-strategy unsafe-best-match
+--system
+"
+
 echo $CMD
 eval $CMD

backend/windmill-worker/src/ansible_executor.rs

@@ -116,6 +116,8 @@ async fn handle_ansible_python_deps(
             job_dir,
             worker_dir,
             &mut Some(occupancy_metrics),
+            true,
+            true,
         )
         .await?;
         additional_python_paths.append(&mut venv_path);

backend/windmill-worker/src/common.rs

@@ -41,7 +41,7 @@ use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
 
 use crate::{
     AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE,
-    MAX_TIMEOUT_DURATION,
+    MAX_TIMEOUT_DURATION, PATH_ENV,
 };
 
 pub async fn build_args_map<'a>(
@@ -860,11 +860,17 @@ pub async fn save_in_cache(
 }
 
 fn tentatively_improve_error(err: Error, executable: &str) -> Error {
+    #[cfg(unix)]
+    let err_msg = "No such file or directory (os error 2)";
+
+    #[cfg(windows)]
+    let err_msg = "program not found";
+
     if err
         .to_string()
-        .contains("No such file or directory (os error 2)")
+        .contains(&err_msg)
     {
-        return Error::InternalErr(format!("Executable {executable} not found on worker"));
+        return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV));
     }
     return err;
 }

backend/windmill-worker/src/global_cache.rs

@@ -20,13 +20,21 @@ use std::sync::Arc;
 pub async fn build_tar_and_push(
     s3_client: Arc<dyn ObjectStore>,
     folder: String,
+    no_uv: bool,
 ) -> error::Result<()> {
     use object_store::path::Path;
 
+    use crate::PY311_CACHE_DIR;
+
     tracing::info!("Started building and pushing piptar {folder}");
     let start = Instant::now();
     let folder_name = folder.split("/").last().unwrap();
-    let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",);
+    let prefix = if no_uv {
+        PIP_CACHE_DIR
+    } else {
+        PY311_CACHE_DIR
+    };
+    let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
 
     let tar_file = std::fs::File::create(&tar_path)?;
     let mut tar = tar::Builder::new(tar_file);
@@ -46,7 +54,10 @@ pub async fn build_tar_and_push(
     // })?;
     if let Err(e) = s3_client
         .put(
-            &Path::from(format!("/tar/pip/{folder_name}.tar")),
+            &Path::from(format!(
+                "/tar/{}/{folder_name}.tar",
+                if no_uv { "pip" } else { "python_311" }
+            )),
             std::fs::read(&tar_path)?.into(),
         )
         .await
@@ -71,7 +82,11 @@ pub async fn build_tar_and_push(
 }
 
 #[cfg(all(feature = "enterprise", feature = "parquet"))]
-pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> error::Result<()> {
+pub async fn pull_from_tar(
+    client: Arc<dyn ObjectStore>,
+    folder: String,
+    no_uv: bool,
+) -> error::Result<()> {
     use windmill_common::s3_helpers::attempt_fetch_bytes;
 
     let folder_name = folder.split("/").last().unwrap();
@@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> erro
     tracing::info!("Attempting to pull piptar {folder_name} from bucket");
 
     let start = Instant::now();
-    let tar_path = format!("tar/pip/{folder_name}.tar");
+    let tar_path = format!(
+        "tar/{}/{folder_name}.tar",
+        if no_uv { "pip" } else { "python_311" }
+    );
     let bytes = attempt_fetch_bytes(client, &tar_path).await?;
 
     // tracing::info!("B: {target} {folder}");

backend/windmill-worker/src/python_executor.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, process::Stdio};
+use std::{collections::HashMap, fs, process::Stdio};
 
 use itertools::Itertools;
 use regex::Regex;
@@ -42,6 +42,10 @@ lazy_static::lazy_static! {
     static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
         .ok().map(|flag| flag == "true").unwrap_or(false);
 
+    /// Use pip install
+    static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
+        .ok().map(|flag| flag == "true").unwrap_or(false);
+
 
     static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
 
@@ -50,6 +54,8 @@ lazy_static::lazy_static! {
 }
 
 const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
+const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str =
+    include_str!("../nsjail/download.py.pip.config.proto");
 const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
 const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
 
@@ -66,8 +72,8 @@ use crate::{
     },
     handle_child::handle_child,
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR,
-    NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV,
-    UV_CACHE_DIR,
+    NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS,
+    PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR,
 };
 
 #[cfg(windows)]
@@ -314,7 +320,7 @@ pub async fn uv_pip_compile(
             .args(args)
             .stdout(Stdio::piped())
             .stderr(Stdio::piped());
-        let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?;
+        let child_process = start_child_process(child_cmd, uv_cmd).await?;
         append_logs(&job_id, &w_id, logs, db).await;
         handle_child(
             job_id,
@@ -901,10 +907,10 @@ async fn handle_python_deps(
         .unwrap_or_else(|| vec![])
         .clone();
 
+    let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content);
     let requirements = match requirements_o {
         Some(r) => r,
         None => {
-            let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content);
             let mut already_visited = vec![];
 
             let requirements = windmill_parser_py_imports::parse_python_imports(
@@ -929,8 +935,8 @@ async fn handle_python_deps(
                     worker_name,
                     w_id,
                     occupancy_metrics,
-                    annotation.no_uv,
-                    annotation.no_cache,
+                    annotations.no_uv || annotations.no_uv_compile,
+                    annotations.no_cache,
                 )
                 .await
                 .map_err(|e| {
@@ -955,6 +961,8 @@ async fn handle_python_deps(
             job_dir,
             worker_dir,
             occupancy_metrics,
+            annotations.no_uv || annotations.no_uv_install,
+            false,
         )
         .await?;
         additional_python_paths.append(&mut venv_path);
@@ -966,6 +974,7 @@ lazy_static::lazy_static! {
     static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap();
 }
 
+/// pip install, include cached or pull from S3
 pub async fn handle_python_reqs(
     requirements: Vec<&str>,
     job_id: &Uuid,
@@ -977,12 +986,22 @@ pub async fn handle_python_reqs(
     job_dir: &str,
     worker_dir: &str,
     occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
+    // TODO: Remove (Deprecated)
+    mut no_uv_install: bool,
+    is_ansible: bool,
 ) -> error::Result<Vec<String>> {
     let mut req_paths: Vec<String> = vec![];
     let mut vars = vec![("PATH", PATH_ENV.as_str())];
     let pip_extra_index_url;
     let pip_index_url;
 
+    no_uv_install |= *USE_PIP_INSTALL;
+
+    if no_uv_install && !is_ansible {
+        append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await;
+        tracing::warn!("Fallback to pip");
+    }
+
     if !*DISABLE_NSJAIL {
         pip_extra_index_url = PIP_EXTRA_INDEX_URL
             .read()
@@ -1013,10 +1032,21 @@ pub async fn handle_python_reqs(
         let _ = write_file(
             job_dir,
             "download.config.proto",
-            &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
-                .replace("{WORKER_DIR}", &worker_dir)
-                .replace("{CACHE_DIR}", PIP_CACHE_DIR)
-                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
+            &(if no_uv_install {
+                NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK
+            } else {
+                NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
+            })
+            .replace("{WORKER_DIR}", &worker_dir)
+            .replace(
+                "{CACHE_DIR}",
+                if no_uv_install {
+                    PIP_CACHE_DIR
+                } else {
+                    PY311_CACHE_DIR
+                },
+            )
+            .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
         )?;
     };
 
@@ -1026,8 +1056,14 @@ pub async fn handle_python_reqs(
         if req.starts_with('#') {
             continue;
         }
+        let py_prefix = if no_uv_install {
+            PIP_CACHE_DIR
+        } else {
+            PY311_CACHE_DIR
+        };
+
         let venv_p = format!(
-            "{PIP_CACHE_DIR}/{}",
+            "{py_prefix}/{}",
             req.replace(' ', "").replace('/', "").replace(':', "")
         );
         if metadata(&venv_p).await.is_ok() {
@@ -1073,7 +1109,10 @@ pub async fn handle_python_reqs(
                 .map(|(req, venv_p)| {
                     let os = os.clone();
                     async move {
-                        if pull_from_tar(os, venv_p.clone()).await.is_ok() {
+                        if pull_from_tar(os, venv_p.clone(), no_uv_install)
+                            .await
+                            .is_ok()
+                        {
                             PullFromTar::Pulled(venv_p.to_string())
                         } else {
                             PullFromTar::NotPulled(req.to_string(), venv_p.to_string())
@@ -1114,7 +1153,11 @@ pub async fn handle_python_reqs(
 
     for (req, venv_p) in req_with_penv {
         let mut logs1 = String::new();
-        logs1.push_str("\n\n--- PIP INSTALL ---\n");
+        if no_uv_install {
+            logs1.push_str("\n\n--- PIP INSTALL ---\n");
+        } else {
+            logs1.push_str("\n\n--- UV PIP INSTALL ---\n");
+        }
         logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
         append_logs(&job_id, w_id, logs1, db).await;
 
@@ -1150,21 +1193,47 @@ pub async fn handle_python_reqs(
             #[cfg(windows)]
             let req = format!("{}", req);
 
-            let mut command_args = vec![
-                PYTHON_PATH.as_str(),
-                "-m",
-                "pip",
-                "install",
-                &req,
-                "-I",
-                "--no-deps",
-                "--no-color",
-                "--isolated",
-                "--no-warn-conflicts",
-                "--disable-pip-version-check",
-                "-t",
-                venv_p.as_str(),
-            ];
+            let mut command_args = if no_uv_install {
+                vec![
+                    PYTHON_PATH.as_str(),
+                    "-m",
+                    "pip",
+                    "install",
+                    &req,
+                    "-I",
+                    "--no-deps",
+                    "--no-color",
+                    "--isolated",
+                    "--no-warn-conflicts",
+                    "--disable-pip-version-check",
+                    "-t",
+                    venv_p.as_str(),
+                ]
+            } else {
+                vec![
+                    UV_PATH.as_str(),
+                    "pip",
+                    "install",
+                    &req,
+                    "--no-deps",
+                    "--no-color",
+                    // "-p",
+                    // "3.11",
+                    // Prevent uv from discovering configuration files.
+                    "--no-config",
+                    "--link-mode=copy",
+                    // TODO: Doublecheck it
+                    "--system",
+                    // Prefer main index over extra
+                    // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
+                    // TODO: Use env variable that can be toggled from UI
+                    "--index-strategy",
+                    "unsafe-best-match",
+                    "--target",
+                    venv_p.as_str(),
+                    "--no-cache",
+                ]
+            };
             let pip_extra_index_url = PIP_EXTRA_INDEX_URL
                 .read()
                 .await
@@ -1196,7 +1265,7 @@ pub async fn handle_python_reqs(
 
             envs.push(("HOME", HOME_ENV.as_str()));
 
-            tracing::debug!("pip install command: {:?}", command_args);
+            tracing::debug!("uv pip install command: {:?}", command_args);
 
             #[cfg(unix)]
             {
@@ -1207,7 +1276,12 @@ pub async fn handle_python_reqs(
                     .envs(envs)
                     .args([
                         "-x",
-                        &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
+                        &format!(
+                            "{}/{}-{}.lock",
+                            LOCK_CACHE_DIR,
+                            if no_uv_install { "pip" } else { "py311" },
+                            fssafe_req
+                        ),
                         "--command",
                         &command_args.join(" "),
                     ])
@@ -1218,16 +1292,20 @@ pub async fn handle_python_reqs(
 
             #[cfg(windows)]
             {
-                let mut pip_cmd = Command::new(PYTHON_PATH.as_str());
-                pip_cmd
-                    .env_clear()
+                let installer_path = if no_uv_install { command_args[0] } else { "uv" };
+                let mut cmd: Command = Command::new(&installer_path);
+                cmd.env_clear()
                     .envs(envs)
                     .envs(PROXY_ENVS.clone())
                     .env("SystemRoot", SYSTEM_ROOT.as_str())
+                    .env(
+                        "TMP",
+                        std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
+                    )
                     .args(&command_args[1..])
                     .stdout(Stdio::piped())
                     .stderr(Stdio::piped());
-                start_child_process(pip_cmd, PYTHON_PATH.as_str()).await?
+                start_child_process(cmd, installer_path).await?
             }
         };
 
@@ -1240,7 +1318,7 @@ pub async fn handle_python_reqs(
             false,
             worker_name,
             &w_id,
-            &format!("pip install {req}"),
+            &format!("uv pip install {req}"),
             None,
             false,
             occupancy_metrics,
@@ -1252,6 +1330,16 @@ pub async fn handle_python_reqs(
             "finished setting up python dependencies {}",
             job_id
         );
+        if child.is_err() {
+            
+            if let Err(e) = fs::remove_dir_all(&venv_p) {
+                tracing::warn!(
+                    workspace_id = %w_id,
+                    "failed to remove cache dir: {:?}",
+                    e
+                );
+            }
+        }
         child?;
 
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -1260,7 +1348,7 @@ pub async fn handle_python_reqs(
                 tracing::warn!("S3 cache not available in the pro plan");
             } else {
                 let venv_p = venv_p.clone();
-                tokio::spawn(build_tar_and_push(os, venv_p));
+                tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install));
             }
         }
         req_paths.push(venv_p);

backend/windmill-worker/src/worker.rs

@@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs");
 pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/");
 
 pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock");
+// Used as fallback now
 pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
+
+// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310");
+pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311");
+// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312");
+// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313");
+
 pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv");
 pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip");
 pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
@@ -257,6 +264,7 @@ const NUM_SECS_PING: u64 = 5;
 const NUM_SECS_READINGS: u64 = 60;
 
 const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
+const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh");
 
 pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900;
 pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days
@@ -311,6 +319,7 @@ lazy_static::lazy_static! {
     .and_then(|x| x.parse::<bool>().ok())
     .unwrap_or(false);
 
+    // pub static ref DISABLE_NSJAIL: bool = false;
     pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL")
         .ok()
         .and_then(|x| x.parse::<bool>().ok())
@@ -740,6 +749,13 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
             "download_deps.py.sh",
             INCLUDE_DEPS_PY_SH_CONTENT,
         );
+
+        // TODO: Remove (Deprecated)
+        let _ = write_file(
+            &worker_dir,
+            "download_deps.py.pip.sh",
+            INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK,
+        );
     }
 
     let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);

backend/windmill-worker/src/worker_lockfiles.rs

@@ -1312,6 +1312,8 @@ async fn python_dep(
             job_dir,
             worker_dir,
             occupancy_metrics,
+            false,
+            false,
         )
         .await;
 

benchmarks/suite_config.json

@@ -33,10 +33,10 @@
   },
   {
     "kind": "bigrawscript",
-    "jobs": 10000
+    "jobs": 4000
   },
   {
     "kind": "bigscriptinflow",
-    "jobs": 10000
+    "jobs": 4000
   }
 ]

frontend/src/lib/components/InstanceSettings.svelte

@@ -14,7 +14,7 @@
 	import Alert from './common/alert/Alert.svelte'
 	import { isCloudHosted } from '$lib/cloud'
 	import { capitalize, classNames, sleep } from '$lib/utils'
-	import { enterpriseLicense } from '$lib/stores'
+	import { enterpriseLicense, isCriticalAlertsUIOpen } from '$lib/stores'
 	import CustomOauth from './CustomOauth.svelte'
 	import {
 		AlertCircle,
@@ -43,6 +43,7 @@
 	export let tab: string = 'Core'
 	export let hideTabs: boolean = false
 	export let hideSave: boolean = false
+	export let closeDrawer: (() => void) | undefined = () => {}
 
 	let values: Record<string, any> = {}
 	let initialOauths: Record<string, any> = {}
@@ -814,6 +815,26 @@
 													placeholder={setting.placeholder}
 													bind:value={values[setting.key]}
 												/>
+											{:else if setting.key == 'critical_alert_mute_ui'}
+												<div class="flex flex-col gap-y-2 my-2 py-2">
+													<Toggle
+														disabled={!$enterpriseLicense}
+														bind:checked={values[setting.key]}
+														options={{ right: setting.description }}
+													/>
+													<div class="flex flex-row">
+														<Button
+															disabled={!$enterpriseLicense}
+															size="sm"
+															on:click={() => {
+																isCriticalAlertsUIOpen.set(true)
+																closeDrawer?.()
+															}}
+														>
+															Show Critical Alerts
+														</Button>
+													</div>
+												</div>
 											{:else if setting.fieldType == 'critical_error_channels'}
 												<div class="w-full flex gap-x-16 flex-wrap">
 													<div class="w-full max-w-lg">

frontend/src/lib/components/SuperadminSettings.svelte

@@ -325,7 +325,7 @@
 							</div>
 						</TabContent>
 						<TabContent value="" values={settingsKeys}>
-							<InstanceSettings bind:this={instanceSettings} hideTabs hideSave {tab} />
+							<InstanceSettings bind:this={instanceSettings} hideTabs hideSave {tab} {closeDrawer}/>
 						</TabContent>
 					</svelte:fragment>
 				</Tabs>

frontend/src/lib/components/common/modal/Modal.svelte

@@ -44,7 +44,7 @@
 	<div
 		on:click={() => (open = false)}
 		transition:fadeFast|local
-		class={'absolute top-0 bottom-0 left-0 right-0 z-50'}
+		class={'absolute top-0 bottom-0 left-0 right-0 z-[9999]'}
 		role="dialog"
 	>
 		<div

frontend/src/lib/components/flows/content/FlowEditorPanel.svelte

@@ -62,7 +62,7 @@
 {:else if $selectedId === 'Input'}
 	<FlowInput {noEditor} disabled={disabledFlowInputs} />
 {:else if $selectedId === 'Result'}
-	<p class="p-4 text-secondary">Nothing to show about the result node. Happy flow building!</p>
+	<p class="p-4 text-secondary">Result of the flow will be the result of the last node.</p>
 {:else if $selectedId === 'constants'}
 	<FlowConstants {noEditor} />
 {:else if $selectedId === 'failure'}

frontend/src/lib/components/instanceSettings.ts

@@ -140,7 +140,7 @@ export const settings: Record<string, Setting[]> = {
 		{
 			label: 'Critical alert channels',
 			description:
-				'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. <a href="https://www.windmill.dev/docs/core_concepts/critical_alert_channels">Learn more</a>',
+				'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. <a href="https://www.windmill.dev/docs/core_concepts/critical_alerts">Learn more</a>',
 			key: 'critical_error_channels',
 			fieldType: 'critical_error_channels',
 			storage: 'setting',

frontend/src/lib/components/sidebar/CriticalAlertButton.svelte

@@ -1,64 +0,0 @@
-<script lang="ts">
-	import { twMerge } from 'tailwind-merge'
-	import Popover from '../Popover.svelte'
-	import { createEventDispatcher } from 'svelte'
-
-	export let label: string | undefined = undefined
-	export let numUnacknowledgedCriticalAlerts: number
-	export let isCollapsed: boolean
-	export let disabled: boolean = false
-	export let lightMode: boolean = false
-	export let stopPropagationOnClick: boolean = false
-	export let shortcut: string = ''
-
-	let dispatch = createEventDispatcher()
-</script>
-
-{#if !disabled}
-	<Popover appearTimeout={0} disappearTimeout={0} class="w-full" disablePopup={!isCollapsed}>
-		<div class="py-1.5 px-1 border-t border-gray-700">
-		<button
-			on:click={(e) => {
-				if (stopPropagationOnClick) e.preventDefault()
-				dispatch('click')
-			}}
-			class={twMerge(
-				'group flex items-center px-2 py-2 font-light rounded-md h-8 gap-3 w-full',
-				lightMode
-					? 'text-primary hover:bg-surface-hover '
-					: '  hover:bg-[#2A3648] text-primary-inverse dark:text-primary',
-				'transition-all',
-				$$props.class
-			)}
-			title={label}
-		>
-			<span
-				class="flex items-center justify-center h-6 w-6 rounded-lg bg-red-600 text-white font-bold text-xs"
-			>
-				{numUnacknowledgedCriticalAlerts > 9 ? '9+' : numUnacknowledgedCriticalAlerts}
-			</span>
-
-			{#if !isCollapsed && label}
-				<span
-					class={twMerge(
-						'whitespace-pre truncate',
-						lightMode ? 'text-primary' : 'text-primary-inverse  dark:text-primary',
-						'transition-all',
-						$$props.class
-					)}
-				>
-					{label}
-					<span class="pl-2 text-xs dark:text-secondary light:text-secondary-inverse font-semibold">
-						{shortcut}
-					</span>
-				</span>
-			{/if}
-		</button>
-	</div>
-		<svelte:fragment slot="text">
-			{#if label}
-				{label}
-			{/if}
-		</svelte:fragment>
-	</Popover>
-{/if}

frontend/src/lib/components/sidebar/CriticalAlertModal.svelte

@@ -3,16 +3,62 @@
 	import CriticalAlertModalInner from './CriticalAlertModalInner.svelte'
 	import { SettingService } from '$lib/gen'
 	import { sendUserToast } from '$lib/toast'
+	import { superadmin, workspaceStore, isCriticalAlertsUIOpen } from '$lib/stores'
 	import Modal from '../common/modal/Modal.svelte'
 
 	export let open: boolean = false
 	export let numUnacknowledgedCriticalAlerts: number = 0
+	export let muteSettings
+	let workspaceContext = false
+
+	$: {
+		setupApiFunctions(workspaceContext)
+	}
+
+	function setupApiFunctions(_ctx?) {
+		getCriticalAlerts = withSuperadminLogic(
+			SettingService.getCriticalAlerts,
+			SettingService.workspaceGetCriticalAlerts
+		)
+
+		acknowledgeCriticalAlert = withSuperadminLogic(
+			SettingService.acknowledgeCriticalAlert,
+			SettingService.workspaceAcknowledgeCriticalAlert
+		)
+
+		acknowledgeAllCriticalAlerts = withSuperadminLogic(
+			SettingService.acknowledgeAllCriticalAlerts,
+			SettingService.workspaceAcknowledgeAllCriticalAlerts
+		)
+	}
+
+	$: isCriticalAlertsUIOpen.set(open)
+	$: if ($isCriticalAlertsUIOpen) open = $isCriticalAlertsUIOpen
 
 	let checkForNewAlertsInterval: ReturnType<typeof setInterval>
 	let checkingForNewAlerts = false
 
-	onMount(() => {
-		updateHasUnacknowledgedCriticalAlerts(true)
+	const withSuperadminLogic = (superadminFunction, workspaceFunction) => {
+		return async (params = {}) => {
+			if (!$superadmin || workspaceContext) {
+				return workspaceFunction({
+					...params,
+					workspace: $workspaceStore
+				})
+			} else {
+				return superadminFunction(params)
+			}
+		}
+	}
+
+	let getCriticalAlerts
+	let acknowledgeCriticalAlert
+	let acknowledgeAllCriticalAlerts
+
+	setupApiFunctions()
+
+	onMount(async () => {
+		await updateHasUnacknowledgedCriticalAlerts(false)
 		checkForNewAlertsInterval = setInterval(() => {
 			updateHasUnacknowledgedCriticalAlerts(true)
 		}, 15000)
@@ -25,15 +71,18 @@
 	async function updateHasUnacknowledgedCriticalAlerts(sendToast: boolean = false) {
 		if (checkingForNewAlerts) return
 		checkingForNewAlerts = true
-
 		try {
-			const unacknowledged = await SettingService.getCriticalAlerts({
+			const unacknowledged = await getCriticalAlerts({
 				page: 1,
 				pageSize: 10,
 				acknowledged: false
 			})
-
-			if (numUnacknowledgedCriticalAlerts === 0 && unacknowledged.length > 0 && sendToast) {
+			if (
+				numUnacknowledgedCriticalAlerts === 0 &&
+				unacknowledged.length > 0 &&
+				sendToast &&
+				(($superadmin && !muteSettings.global) || (!$superadmin && !muteSettings.workspace))
+			) {
 				sendUserToast(
 					'Critical Alert:',
 					true,
@@ -62,11 +111,19 @@
 	}
 
 	async function acknowledgeAlert(id: number) {
-		await SettingService.acknowledgeCriticalAlert({ id })
+		await acknowledgeCriticalAlert({ id })
 		updateHasUnacknowledgedCriticalAlerts()
 	}
 </script>
 
 <Modal bind:open title="Critical Alerts" cancelText="Close" style="max-width: 66%;">
-	<CriticalAlertModalInner {updateHasUnacknowledgedCriticalAlerts} />
+	<CriticalAlertModalInner
+		{numUnacknowledgedCriticalAlerts}
+		{updateHasUnacknowledgedCriticalAlerts}
+		{getCriticalAlerts}
+		{acknowledgeCriticalAlert}
+		{acknowledgeAllCriticalAlerts}
+		{muteSettings}
+		bind:workspaceContext
+	/>
 </Modal>

frontend/src/lib/components/sidebar/CriticalAlertModalInner.svelte

@@ -2,23 +2,77 @@
 	import Button from '../common/button/Button.svelte'
 	import Toggle from '$lib/components/Toggle.svelte'
 	import { SettingService } from '$lib/gen'
-	import { CheckCircle2, AlertCircle, RefreshCw, CheckSquare2, AlertTriangle } from 'lucide-svelte'
+	import {
+		CheckCircle2,
+		AlertCircle,
+		RefreshCw,
+		CheckSquare2,
+		AlertTriangle,
+	} from 'lucide-svelte'
 	import type { CriticalAlert } from '$lib/gen'
 	import { onMount } from 'svelte'
 	import { instanceSettingsSelectedTab } from '$lib/stores'
 	import { goto } from '$app/navigation'
+	import { superadmin, workspaceStore } from '$lib/stores'
+	import { sendUserToast } from '$lib/toast'
+	import Section from '$lib/components/Section.svelte'
 
-	export let updateHasUnacknowledgedCriticalAlerts: () => void = () => {}
+	export let updateHasUnacknowledgedCriticalAlerts
+	export let getCriticalAlerts
+	export let acknowledgeCriticalAlert
+	export let acknowledgeAllCriticalAlerts
+	export let numUnacknowledgedCriticalAlerts
 
 	let alerts: CriticalAlert[] = []
 
 	let isRefreshing = false
-	let hasCriticalAlertChannels = false
+	let hasCriticalAlertChannels = true
+
+	export let muteSettings = {
+		workspace: true,
+		global: true
+	}
+
+	$: muteSettings
+	$: {
+		if (initialMuteSettings.workspace !== muteSettings.workspace || initialMuteSettings.global !== muteSettings.global) {
+			saveMuteSettings()
+		}
+	}
+
+	$: numUnacknowledgedCriticalAlerts >= 0 && getAlerts(true)
+
+	let initialMuteSettings = muteSettings
+
+	async function saveMuteSettings() {
+		if (initialMuteSettings.workspace !== muteSettings.workspace) {
+			// Workspace
+			await SettingService.workspaceMuteCriticalAlertsUi({
+				workspace: $workspaceStore!,
+				requestBody: {
+					mute_critical_alerts: muteSettings.workspace
+				}
+			})
+		}
+		if ($superadmin && initialMuteSettings.global !== muteSettings.global) {
+			// Global
+			await SettingService.setGlobal({
+				key: 'critical_alert_mute_ui',
+				requestBody: { value: muteSettings.global }
+			})
+		}
+		sendUserToast(
+			`Critical alert UI mute settings changed.\nPlease reload page for UI changes to take effect.`
+		)
+		getAlerts(true)
+		initialMuteSettings = { ...muteSettings }
+	}
 
 	$: loading = isRefreshing
 
 	onMount(() => {
 		refreshAlerts()
+		initialMuteSettings = { ...muteSettings }
 	})
 
 	// Pagination
@@ -29,14 +83,14 @@
 	let hideAcknowledged = false
 
 	async function acknowledgeAll() {
-		await SettingService.acknowledgeAllCriticalAlerts()
+		await acknowledgeAllCriticalAlerts()
 		getAlerts(false)
 	}
 
 	async function fetchAlerts(pageNumber: number) {
 		isRefreshing = true
 		try {
-			const newAlerts = await SettingService.getCriticalAlerts({
+			const newAlerts = await getCriticalAlerts({
 				page: pageNumber,
 				pageSize: pageSize,
 				acknowledged: hideAcknowledged ? false : undefined
@@ -55,6 +109,7 @@
 
 	async function getAlerts(reset?: boolean) {
 		if (reset) page = 1
+		updateHasUnacknowledgedCriticalAlerts()
 		await fetchAlerts(page)
 	}
 
@@ -64,7 +119,7 @@
 	}
 
 	async function acknowledgeAlert(id: number) {
-		await SettingService.acknowledgeCriticalAlert({ id })
+		await acknowledgeCriticalAlert({ id })
 		getAlerts(false)
 	}
 
@@ -82,7 +137,7 @@
 	}
 
 	async function refreshAlerts() {
-		checkCriticalAlertChannels()
+		if ($superadmin) checkCriticalAlertChannels()
 		await getAlerts(true)
 	}
 
@@ -102,46 +157,99 @@
 		goto('/#superadmin-settings')
 		instanceSettingsSelectedTab.set('Core')
 	}
+
+	export let workspaceContext = false
+
+	$: {
+		workspaceContextChanged(workspaceContext)
+	}
+
+	async function workspaceContextChanged(_ctx) {
+		await getAlerts(true)
+	}
 </script>
 
 <div>
-	{#if !hasCriticalAlertChannels}
-		<div class="flex flex-row pb-4">
-			<AlertTriangle color="orange" class="w-6 h-6 mr-2" />
-			<p>
-				No critical alert channels are set up. Go to the
-				<a href="/#superadmin-settings" on:click|preventDefault={goToCoreTab}>Instance Settings</a>
-				page to configure critical alert channels.
-			</p>
+	<div class="grid grid-cols-3 gap-4 col-start-3">
+		<div class="pt-1 col-span-2">
+			{#if !hasCriticalAlertChannels && $superadmin}
+				<div class="flex flex-row pb-4">
+					<AlertTriangle color="orange" class="w-6 h-6 mr-2" />
+					<p>
+						No critical alert channels are set up. Go to the
+						<a href="/#superadmin-settings" on:click|preventDefault={goToCoreTab}
+							>Instance Settings</a
+						>
+						page to configure critical alert channels.
+					</p>
+				</div>
+			{/if}
 		</div>
-	{/if}
-	<!-- Row of action buttons above the table -->
-	<div class="flex justify-between items-center mb-4">
-		<div class="flex space-x-2">
-			<Button color="green" startIcon={{ icon: CheckSquare2 }} size="sm" on:click={acknowledgeAll}
-				>Acknowledge All</Button
+
+		<div class="flex flex-col justify-between col-start-3">
+			<div class="flex flex-row justify-end mt-[-38px] pb-3">
+				<Button
+					color="green"
+					startIcon={{ icon: CheckSquare2 }}
+					size="xs"
+					disabled={numUnacknowledgedCriticalAlerts === 0}
+					on:click={acknowledgeAll}
+				>
+					Acknowledge All</Button
+				>
+			</div>
+
+			{#if $superadmin}
+				<div class="flex flex-row py-2 pb-3">
+					<Toggle
+						bind:checked={workspaceContext}
+						options={{ right: `Workspace: '${$workspaceStore}'`, left: "Context: 'Global'" }}
+						size="xs"
+					/>
+				</div>
+			{/if}
+
+			<Section
+				label="Mute Settings"
+				collapsable={true}
+				small={true}
 			>
-		</div>
+				{#if $superadmin}
+					<div class="flex flex-row pb-1">
+						<Toggle
+							bind:checked={muteSettings.global}
+							options={{ right: 'Mute critical alerts instance wide' }}
+							size="xs"
+						/>
+					</div>
+				{/if}
 
-		<button class="p-2 rounded-full hover:bg-gray-200" on:click={refreshAlerts} disabled={loading}>
-			<RefreshCw class={loading ? 'animate-spin ' : ''} size="20" />
-		</button>
-	</div>
+				<div class="flex flex-row pb-1">
+					<Toggle
+						bind:checked={muteSettings.workspace}
+						options={{ right: 'Mute critical alerts for current workspace' }}
+						size="xs"
+					/>
+				</div>
+			</Section>
 
-	<!-- Pagination controls above the table -->
-	<div class="flex justify-between items-center mb-2">
-		<div class="flex items-center space-x-4">
-			<Button size="xs2" on:click={goToPreviousPage} disabled={page <= 1}>Previous</Button>
-			<span>Page {page}</span>
-			<Button size="xs2" on:click={goToNextPage} disabled={!hasMore}>Next</Button>
-		</div>
-		<div class="pr-3">
-			<Toggle
-				bind:checked={hideAcknowledged}
-				on:change={refreshAlerts}
-				options={{ right: 'Hide Acknowledged' }}
-				size="xs"
-			/>
+			<div class="pt-2 flex justify-between items-center">
+				<div class="pr-2">
+					<Toggle
+						bind:checked={hideAcknowledged}
+						on:change={refreshAlerts}
+						options={{ right: 'Hide Acknowledged' }}
+						size="xs"
+					/>
+				</div>
+				<button
+					class="mb-1 p-2 rounded-full hover:bg-gray-200"
+					on:click={refreshAlerts}
+					disabled={loading}
+				>
+					<RefreshCw class={loading ? 'animate-spin ' : ''} size="20" />
+				</button>
+			</div>
 		</div>
 	</div>
 
@@ -153,11 +261,14 @@
 					<th class="w-[60px] px-4 py-2 text-center">Type</th>
 					<th class="px-4 py-2 text-center">Message</th>
 					<th class="w-[150px] px-4 py-2 text-center">Created At</th>
+					{#if $superadmin}
+						<th class="w-[80px] px-4 py-2 text-center">Workspace</th>
+					{/if}
 					<th class="w-[180px] px-4 py-2 text-center">Acknowledge</th>
 				</tr>
 			</thead>
 			<tbody>
-				{#each alerts as { id, alert_type, message, created_at, acknowledged }}
+				{#each alerts as { id, alert_type, message, created_at, acknowledged, workspace_id }}
 					{#if !hideAcknowledged || !acknowledged}
 						<tr class="bg-gray-100 dark:bg-gray-700 dark:text-white text-center">
 							<td class="border px-4 py-2 w-[100px]">
@@ -174,6 +285,9 @@
 							<td class="border px-4 py-2">{message}</td>
 							<!-- Flexible width -->
 							<td class="border px-4 py-2 w-[150px]">{formatDate(created_at)}</td>
+							{#if $superadmin}
+								<td class="border px-4 py-2 w-[150px]">{workspace_id ? workspace_id : 'global'}</td>
+							{/if}
 							<td class="border px-4 py-2 w-[180px]">
 								<div class="flex justify-center items-center">
 									{#if !acknowledged}
@@ -196,6 +310,11 @@
 			</tbody>
 		</table>
 	</div>
+	<div class="flex flex-1 pt-2 gap-x-4 justify-end">
+		<Button size="xs2" on:click={goToPreviousPage} disabled={page <= 1}>Previous</Button>
+		<span>Page {page}</span>
+		<Button size="xs2" on:click={goToNextPage} disabled={!hasMore}>Next</Button>
+	</div>
 
 	{#if alerts.length === 0}
 		<p class="text-center text-gray-500 mt-4">No critical alerts available.</p>

frontend/src/lib/components/sidebar/MenuButton.svelte

@@ -2,14 +2,15 @@
 	import { twMerge } from 'tailwind-merge'
 	import Popover from '../Popover.svelte'
 	import { createEventDispatcher } from 'svelte'
-
+	import SideBarNotification from './SideBarNotification.svelte'
 	export let label: string | undefined = undefined
 	export let icon: any | undefined = undefined
 	export let isCollapsed: boolean
 	export let disabled: boolean = false
 	export let lightMode: boolean = false
 	export let stopPropagationOnClick: boolean = false
-	export let shortcut: string = ""
+	export let shortcut: string = ''
+	export let notificationsCount: number = 0
 
 	let dispatch = createEventDispatcher()
 </script>
@@ -60,6 +61,16 @@
 					</span>
 				</span>
 			{/if}
+
+			{#if isCollapsed && notificationsCount > 0}
+				<div class="absolute translate-x-1/2 translate-y-1/2 -top-2 right-1 flex h-fit w-fit">
+					<SideBarNotification notificationCount={notificationsCount} small={true} />
+				</div>
+			{:else if notificationsCount > 0}
+				<div class="ml-auto">
+					<SideBarNotification notificationCount={notificationsCount} small={false} />
+				</div>
+			{/if}
 		</button>
 		<svelte:fragment slot="text">
 			{#if label}

frontend/src/lib/components/sidebar/SideBarNotification.svelte

@@ -0,0 +1,14 @@
+<script lang="ts">
+	export let notificationCount = 0
+	export let small: boolean = false
+</script>
+
+{#if !small}
+	<div
+		class="bg-red-500 text-white text-[0.6rem] rounded-md w-5 h-5 flex items-center justify-center"
+	>
+		{notificationCount > 9 ? '9+' : notificationCount}
+	</div>
+{:else}
+	<div class="bg-red-500 rounded-md w-3 h-3 flex items-center justify-center" />
+{/if}

frontend/src/lib/components/sidebar/SidebarContent.svelte

@@ -1,6 +1,13 @@
 <script lang="ts">
 	import MenuLink from './MenuLink.svelte'
-	import { superadmin, usedTriggerKinds, userStore, workspaceStore } from '$lib/stores'
+	import {
+		superadmin,
+		usedTriggerKinds,
+		userStore,
+		workspaceStore,
+		isCriticalAlertsUIOpen,
+		enterpriseLicense
+	} from '$lib/stores'
 	import { SIDEBAR_SHOW_SCHEDULES } from '$lib/consts'
 	import {
 		BookOpen,
@@ -23,7 +30,8 @@
 		Settings,
 		UserCog,
 		Plus,
-		Unplug
+		Unplug,
+		AlertCircle
 	} from 'lucide-svelte'
 	import Menu from '../common/menu/MenuV2.svelte'
 	import MenuButton from './MenuButton.svelte'
@@ -40,6 +48,9 @@
 	import { base } from '$lib/base'
 	import { type Changelog, changelogs } from './changelogs'
 	import { page } from '$app/stores'
+	import SideBarNotification from './SideBarNotification.svelte'
+
+	export let numUnacknowledgedCriticalAlerts = 0
 
 	$: mainMenuLinks = [
 		{ label: 'Home', href: `${base}/`, icon: Home },
@@ -168,14 +179,8 @@
 			],
 			disabled: $userStore?.operator
 		},
-		!$superadmin
+		$superadmin || $userStore?.is_admin
 			? {
-					label: 'Audit Logs',
-					href: `${base}/audit_logs`,
-					icon: Eye,
-					disabled: $userStore?.operator
-			  }
-			: {
 					label: 'Logs',
 					icon: Logs,
 					subItems: [
@@ -184,12 +189,34 @@
 							href: `${base}/audit_logs`,
 							icon: Eye
 						},
-						{
-							label: 'Service Logs',
-							href: `${base}/service_logs`,
-							icon: Logs
+						...($superadmin
+							? [
+									{
+										label: 'Service Logs',
+										href: `${base}/service_logs`,
+										icon: Logs
+									}
+							  ]
+							: []),
+						...($enterpriseLicense
+							? [
+									{
+										label: 'Critical Alerts',
+										action: () => {
+											isCriticalAlertsUIOpen.set(true)
+										},
+							icon: AlertCircle,
+							notificationCount: numUnacknowledgedCriticalAlerts
 						}
 					]
+							: [])
+					]
+			  }
+			: {
+					label: 'Audit Logs',
+					href: `${base}/audit_logs`,
+					icon: Eye,
+					disabled: $userStore?.operator
 			  }
 	]
 
@@ -243,6 +270,14 @@
 	export let noGap: boolean = false
 
 	let leaveWorkspaceModal = false
+
+	function computeAllNotificationsCount(menuItems: any[]) {
+		let count = 0
+		for (const menuItem of menuItems) {
+			count += menuItem?.['notificationCount'] ?? 0
+		}
+		return count
+	}
 </script>
 
 <nav
@@ -306,16 +341,17 @@
 			<UserMenu {isCollapsed} />
 			{#each secondaryMenuLinks as menuLink (menuLink.href ?? menuLink.label)}
 				{#if menuLink.subItems}
+					{@const notificationsCount = computeAllNotificationsCount(menuLink.subItems)}
 					<Menu>
 						<div slot="trigger">
-							<MenuButton class="!text-2xs" {...menuLink} {isCollapsed} />
+							<MenuButton class="!text-2xs" {...menuLink} {isCollapsed} {notificationsCount} />
 						</div>
 						{#each menuLink.subItems as subItem (subItem.href ?? subItem.label)}
 							<MenuItem>
 								<div class="py-1" role="none">
 									{#if subItem?.['action']}
 										<button
-											class="text-secondary block px-4 py-2 text-xs hover:bg-surface-hover hover:text-primary"
+											class="text-secondary font-normal w-full block px-4 py-2 text-2xs hover:bg-surface-hover hover:text-primary"
 											on:click={subItem?.['action']}
 										>
 											<div class="flex flex-row items-center gap-2">
@@ -324,14 +360,17 @@
 												{/if}
 
 												{subItem.label}
+												{#if subItem?.['notificationCount']}
+													<div class="ml-auto">
+														<SideBarNotification notificationCount={subItem['notificationCount']} />
+													</div>
+												{/if}
 											</div>
 										</button>
 									{:else}
 										<a
 											href={subItem.href}
-											class={twMerge(
-												'text-secondary block px-4 py-2 text-2xs hover:bg-surface-hover hover:text-primary'
-											)}
+											class="text-secondary font-normal block px-4 py-2 text-2xs hover:bg-surface-hover hover:text-primary"
 											role="menuitem"
 											tabindex="-1"
 										>
@@ -341,6 +380,12 @@
 												{/if}
 
 												{subItem.label}
+
+												{#if subItem?.['notificationCount']}
+													<div class="ml-auto">
+														<SideBarNotification notificationCount={subItem['notificationCount']} />
+													</div>
+												{/if}
 											</div>
 										</a>
 									{/if}

frontend/src/lib/stores.ts

@@ -124,4 +124,6 @@ export type DBSchemas = Partial<Record<string, DBSchema>>
 
 export const dbSchemas = writable<DBSchemas>({})
 
-export const instanceSettingsSelectedTab = writable('Core')
+export const instanceSettingsSelectedTab = writable('Core')
+
+export const isCriticalAlertsUIOpen = writable(false)

frontend/src/routes/(root)/(logged)/+layout.svelte

@@ -50,7 +50,6 @@
 	import MenuButton from '$lib/components/sidebar/MenuButton.svelte'
 	import { setContext } from 'svelte'
 	import { base } from '$app/paths'
-	import CriticalAlertButton from '$lib/components/sidebar/CriticalAlertButton.svelte'
 
 	OpenAPI.WITH_CREDENTIALS = true
 	let menuOpen = false
@@ -282,23 +281,35 @@
 	setContext('openSearchWithPrefilledText', openSearchModal)
 
 	$: {
-		if ($enterpriseLicense && $superadmin) {
+		if ($enterpriseLicense && $workspaceStore && $superadmin !== undefined && $userStore) {
+			mountModal = true
 			loadCriticalAlertsMuted()
 		}
 	}
 
 	let numUnacknowledgedCriticalAlerts = 0
-	let isCriticalAlertsModalOpen = false
-	let isCriticalAlertsUiMuted = false
-
-	async function loadCriticalAlertsMuted() {
-		isCriticalAlertsUiMuted = (await SettingService.getGlobal({
-			key: 'critical_alert_mute_ui'
-		})) as boolean
+	let mountModal = false
+	let isCriticalAlertsUiMuted = true
+	let muteSettings = {
+		global: true,
+		workspace: true
 	}
+	async function loadCriticalAlertsMuted() {
+		let g_muted = true
+		const ws_muted =
+			(await WorkspaceService.getSettings({ workspace: $workspaceStore! })).mute_critical_alerts ||
+			false
 
-	function openCriticalAlertsModal(text?: string): void {
-		isCriticalAlertsModalOpen = true
+		if ($superadmin) {
+			g_muted = (await SettingService.getGlobal({
+				key: 'critical_alert_mute_ui'
+			})) as boolean
+			isCriticalAlertsUiMuted = g_muted
+		} else {
+			isCriticalAlertsUiMuted = ws_muted
+		}
+
+		muteSettings = { global: g_muted, workspace: ws_muted }
 	}
 </script>
 
@@ -316,14 +327,11 @@
 {:else if $userStore}
 	<GlobalSearchModal bind:this={globalSearchModal} />
 	{#if $superadmin}
-		{#if !isCriticalAlertsUiMuted && $enterpriseLicense}
-			<CriticalAlertModal
-				bind:open={isCriticalAlertsModalOpen}
-				bind:numUnacknowledgedCriticalAlerts
-			/>
-		{/if}
 		<SuperadminSettings bind:this={superadminSettings} />
 	{/if}
+	{#if mountModal}
+		<CriticalAlertModal bind:muteSettings bind:numUnacknowledgedCriticalAlerts />
+	{/if}
 	<div>
 		{#if !menuHidden}
 			{#if !$userStore?.operator}
@@ -401,7 +409,12 @@
 										/>
 									</div>
 
-									<SidebarContent isCollapsed={false} />
+									<SidebarContent
+										isCollapsed={false}
+										numUnacknowledgedCriticalAlerts={isCriticalAlertsUiMuted
+											? 0
+											: numUnacknowledgedCriticalAlerts}
+									/>
 								</div>
 							</div>
 						</div>
@@ -435,17 +448,6 @@
 									{/if}
 								</div>
 							</button>
-							{#if $superadmin && $enterpriseLicense}
-								<CriticalAlertButton
-									stopPropagationOnClick={true}
-									on:click={() => openCriticalAlertsModal()}
-									{numUnacknowledgedCriticalAlerts}
-									{isCollapsed}
-									label="Critical Alerts"
-									class="!text-xs"
-									disabled={numUnacknowledgedCriticalAlerts === 0}
-								/>
-							{/if}
 							<div class="px-2 py-4 space-y-2 border-y border-gray-700">
 								<WorkspaceMenu {isCollapsed} />
 								<FavoriteMenu {favoriteLinks} {isCollapsed} />
@@ -460,7 +462,12 @@
 								/>
 							</div>
 
-							<SidebarContent {isCollapsed} />
+							<SidebarContent
+								{isCollapsed}
+								numUnacknowledgedCriticalAlerts={isCriticalAlertsUiMuted
+									? 0
+									: numUnacknowledgedCriticalAlerts}
+							/>
 
 							<div class="flex-shrink-0 flex px-4 pb-3.5">
 								<button
@@ -548,7 +555,12 @@
 								/>
 							</div>
 
-							<SidebarContent {isCollapsed} />
+							<SidebarContent
+								{isCollapsed}
+								numUnacknowledgedCriticalAlerts={isCriticalAlertsUiMuted
+									? 0
+									: numUnacknowledgedCriticalAlerts}
+							/>
 						</div>
 					</div>
 				</div>

frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte

@@ -15,7 +15,13 @@
 	import Tooltip from '$lib/components/Tooltip.svelte'
 	import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte'
 	import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts'
-	import { OauthService, WorkspaceService, JobService, ResourceService } from '$lib/gen'
+	import {
+		OauthService,
+		WorkspaceService,
+		JobService,
+		ResourceService,
+		SettingService
+	} from '$lib/gen'
 	import {
 		enterpriseLicense,
 		copilotInfo,
@@ -23,7 +29,8 @@
 		userStore,
 		usersWorkspaceStore,
 		workspaceStore,
-		hubBaseUrlStore
+		hubBaseUrlStore,
+		isCriticalAlertsUIOpen
 	} from '$lib/stores'
 	import { sendUserToast } from '$lib/toast'
 	import { emptyString, tryEvery } from '$lib/utils'
@@ -102,6 +109,8 @@
 	let errorHandlerItemKind: 'flow' | 'script' = 'script'
 	let errorHandlerExtraArgs: Record<string, any> = {}
 	let errorHandlerMutedOnCancel: boolean | undefined = undefined
+	let criticalAlertUIMuted: boolean | undefined = undefined
+	let initialCriticalAlertUIMuted: boolean | undefined = undefined
 	let aiResourceInitialPath: string | undefined = undefined
 	let aiResourceInitialProvider: string | undefined = undefined
 
@@ -391,6 +400,8 @@
 		errorHandlerScriptPath = (settings.error_handler ?? '').split('/').slice(1).join('/')
 		errorHandlerInitialScriptPath = errorHandlerScriptPath
 		errorHandlerMutedOnCancel = settings.error_handler_muted_on_cancel
+		criticalAlertUIMuted = settings.mute_critical_alerts
+		initialCriticalAlertUIMuted = settings.mute_critical_alerts
 		if (emptyString($enterpriseLicense)) {
 			errorHandlerSelected = 'custom'
 		} else {
@@ -587,6 +598,22 @@
 			timeout: 5000
 		})
 	}
+
+	async function editCriticalAlertMuteSetting() {
+		await SettingService.workspaceMuteCriticalAlertsUi({
+			workspace: $workspaceStore!,
+			requestBody: {
+				mute_critical_alerts: criticalAlertUIMuted
+			}
+		})
+		sendUserToast(
+			`Critical alert UI mute setting for workspace is set to ${criticalAlertUIMuted}\nreloading page...`
+		)
+		// reload page after change of setting
+		setTimeout(() => {
+			window.location.reload()
+		}, 3000)
+	}
 </script>
 
 <Portal name="workspace-settings">
@@ -972,6 +999,41 @@
 					Save
 				</Button>
 			</div>
+			<div class="flex flex-col gap-4 my-8">
+				<div class="flex flex-col gap-1">
+					<div class=" text-primary text-lg font-semibold"> Workspace Critical Alerts</div>
+					<div class="text-tertiary text-xs">
+						Critical alerts within the scope of a workspace are sent to the workspace admins through
+						a UI notification.
+						<a
+							href="https://www.windmill.dev/docs/core_concepts/critical_alerts"
+							target="_blank"
+							class="text-blue-500">Learn more</a
+						>.
+					</div>
+					<div class="flex flex-col mt-5 gap-5 items-start">
+						<Button
+							disabled={!$enterpriseLicense}
+							size="sm"
+							on:click={() => isCriticalAlertsUIOpen.set(true)}
+						>
+							Show Critical Alerts
+						</Button>
+						<Toggle
+							disabled={!$enterpriseLicense}
+							bind:checked={criticalAlertUIMuted}
+							options={{ right: 'Mute critical alerts UI for this workspace' }}
+						/>
+						<Button
+							disabled={!$enterpriseLicense || criticalAlertUIMuted == initialCriticalAlertUIMuted}
+							size="sm"
+							on:click={editCriticalAlertMuteSetting}
+						>
+							Save Mute Setting
+						</Button>
+					</div>
+				</div>
+			</div>
 		{:else if tab == 'ai'}
 			<div class="flex flex-col gap-4 my-8">
 				<div class="flex flex-col gap-1">

package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "windmill",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {}
+}