nits

* feat: critical alert if disk near full

* update logic to cover edge-case

* update logic

* windows support

* Update README.md

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* make it periodically

* add extra safety

* respect killpills

* do not check every 3 secs

* move to monitor.rs

* rework

* alter readme

* extending functionality

* fix worker_mode

* fix compilation

* fix typo

* make use of AI suggestion

* update ee-repo-ref

* update ee ref

* logs are CE

* update ee repo ref

* remove systemstat from worker crate

* fix comp error + sync cargo.lock

* more comptime fixing

* Update ee-repo-ref.txt

* fix compilation error

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

README.md

@@ -363,6 +363,7 @@ you to have it being synced automatically everyday.
 | DENO_AUTH_TOKENS                    | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
 | DISABLE_RESPONSE_LOGS               | false                  | Disable response logs                                                                                                                                                                              | Server                |
 | CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true                   | If true, only superadmins can create new workspaces                                                                                                                                                | Server                |
+| MIN_FREE_DISK_SPACE_MB              | 15000                  | Minimum amount of free space on worker. Sends critical alert if worker has less free space.                                                                                                        | Worker                |
 
 ## Run a local dev setup
 

backend/.sqlx/query-1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE script SET ws_error_handler_muted = $3 WHERE workspace_id = $2 AND path = $1 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Bool"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "1182fe055306d7ea435d76b74d781e066915c8397e6bbc9e408ff3dda9fec27f"
-}

backend/.sqlx/query-2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))",
+  "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "ea2b88dc050aec038641ea37399d68d4385c5bdc721d1351609f27ca45e4dbdc"
+  "hash": "2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4"
 }

backend/.sqlx/query-726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by created_at DESC",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC",
   "describe": {
     "columns": [
       {
@@ -25,5 +25,5 @@
       true
     ]
   },
-  "hash": "362419eb262c83d6a98a0200b116e831ada60399fe5f55a56d930cc69aff2675"
+  "hash": "726e956cfcd3ac7c07abeecdf92cf0996efe7fa7b671ac2b3b000ead0ea307de"
 }

backend/.sqlx/query-848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE script \n                SET ws_error_handler_muted = $3 \n                WHERE ctid = (\n                    SELECT ctid FROM script\n                    WHERE path = $1 AND workspace_id = $2\n                    ORDER BY created_at DESC\n                    LIMIT 1\n                )\n",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "848c8371eeb17ebd4b36a33f7d8a61eb8f07c54d291bb857ddd41a549cbc88dd"
+}

backend/.sqlx/query-b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND workspace_id = $2)",
+  "query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1",
   "describe": {
     "columns": [
       {
@@ -19,5 +19,5 @@
       false
     ]
   },
-  "hash": "a17260a1f1ee02e786690994d98c84ddf81e2eeb883f895c9cfc47e144d422cb"
+  "hash": "b5860f6a7672a368d740dcd367a8d5ab98fa93e0382a57a698564695db6c40ac"
 }

backend/.sqlx/query-cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg \n        FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n        WHERE s.workspace_id = $1 AND s.path = $2\n        ORDER by s.created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hash",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "deployment_msg",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "cf2a6ad6471a40b6298775cda9300aeecdd75503bed59d80cd62091d1642d1ec"
+}

backend/Cargo.toml

@@ -132,6 +132,8 @@ quote.workspace = true
 memchr.workspace = true
 v8 = { workspace = true, optional = true }
 rustls.workspace = true
+systemstat.workspace = true
+size.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = { optional = true, workspace = true }
@@ -192,7 +194,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
 uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^2"
 anyhow = "^1"
-chrono = { version = "=0.4.39", features = ["serde"] }
+chrono = { version = "^0.4", features = ["serde"] }
 chrono-tz = "^0.10.1"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
@@ -315,9 +317,9 @@ nix = { version = "0.27.1", features = ["process", "signal"] }
 tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" }
 hf-hub = "0.3.2"
 tokenizers = "0.14.1"
-candle-core = "0.3.0"
-candle-transformers = "0.3.0"
-candle-nn = "0.3.0"
+candle-core = "0.9.1"
+candle-transformers = "0.9.1"
+candle-nn = "0.9.1"
 tiberius = { version = "0.12.3", default-features = false, features = ["rustls", "tds73", "chrono", "sql-browser-tokio"]}
 pin-project = "1"
 indexmap = { version = "2.2.5", features = ["serde"]}
@@ -331,8 +333,8 @@ async-nats = "0.38.0"
 nkeys = "0.4.4"
 nu-parser = { version = "0.101.0", default-features = false }
 
-datafusion = "39.0.0"
-object_store = { version = "0.10.0", features = ["aws", "azure"] }
+datafusion = "47.0.0"
+object_store = { version = "0.12.0", features = ["aws", "azure"] }
 openidconnect = { version = "4.0.0-rc.1" }
 aws-config = "^1"
 aws-sdk-sqs = "1.57.0"
@@ -355,9 +357,6 @@ bollard = "0.18.1"
 tonic = { version = "=0.12.3", features = ["tls-native-roots"] }
 byteorder = "1.5.0"
 
-# todo remove
-half = "=2.4.1"
-
 tikv-jemallocator = { version = "0.5" }
 tikv-jemalloc-sys  = { version = "^0.5" }
 tikv-jemalloc-ctl = { version = "^0.5" }
@@ -368,6 +367,8 @@ pin-project-lite = "^0"
 tantivy = "0.22.0"
 
 backon = "1.3.0"
+systemstat = "0.2.4"
+size = "0.5.0"
 
 flume = { version = "0.11.1", features = ["async"] }
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-96a2129f585a1bc6567ef08bd27aaa4aca70543f
+868ccad87afb804fe22818ecd3d5a091199bcdbf

backend/migrations/20250424144434_critical_alerts_on_db_oversize.down.sql

@@ -0,0 +1 @@
+DELETE FROM global_settings WHERE name = 'critical_alerts_on_db_oversize';

backend/migrations/20250424144434_critical_alerts_on_db_oversize.up.sql

@@ -0,0 +1,3 @@
+INSERT INTO global_settings (name, value) 
+VALUES ('critical_alerts_on_db_oversize', '{}') 
+ON CONFLICT (name) DO NOTHING;

backend/migrations/20250506151818_orderby_refactor.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20250506151818_orderby_refactor.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+DROP INDEX IF EXISTS index_script_on_path_created_at;
+CREATE INDEX IF NOT EXISTS index_script_on_path_created_at ON script (workspace_id, path, created_at DESC);

backend/src/main.rs

@@ -8,11 +8,12 @@
 
 use anyhow::Context;
 use monitor::{
-    load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
+    load_base_url, load_otel, reload_critical_alerts_on_db_oversize,
+    reload_delete_logs_periodically_setting, reload_indexer_config,
     reload_instance_python_version_setting, reload_maven_repos_setting,
     reload_no_default_maven_setting, reload_nuget_config_setting,
     reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
-    send_logs_to_object_store,
+    send_logs_to_object_store, WORKERS_NAMES,
 };
 use rand::Rng;
 use sqlx::postgres::PgListener;
@@ -33,15 +34,16 @@ use windmill_common::{
     agent_workers::build_agent_http_client,
     get_database_url,
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
-        DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
-        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
-        NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING,
-        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
+        CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING,
+        ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
+        INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
+        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
         TIMEOUT_WAIT_RESULT_SETTING,
@@ -917,6 +919,12 @@ Windmill Community Edition {GIT_VERSION}
                                                                 tracing::error!(error = %e, "Could not reload critical error emails setting");
                                                             }
                                                         },
+                                                        CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING => {
+                                                            if let Err(e) = reload_critical_alerts_on_db_oversize(&db).await {
+                                                                tracing::error!(error = %e, "Could not reload critical alerts on db oversize setting");
+                                                            }
+
+                                                        },
                                                         JWT_SECRET_SETTING => {
                                                             if let Err(e) = reload_jwt_secret_setting(&db).await {
                                                                 tracing::error!(error = %e, "Could not reload jwt secret setting");
@@ -1214,10 +1222,12 @@ pub async fn run_workers(
         "Starting {num_workers} workers and SLEEP_QUEUE={}ms",
         *windmill_worker::SLEEP_QUEUE
     );
+
     for i in 1..(num_workers + 1) {
         let wk_conf = &workers[i as usize - 1];
         let conn1 = wk_conf.conn.clone();
         let worker_name = wk_conf.worker_name.clone();
+        WORKERS_NAMES.write().await.push(worker_name.clone());
         let ip = ip.clone();
         let rx = killpill_rxs.pop().unwrap();
         let tx = tx.clone();

backend/src/monitor.rs

@@ -13,7 +13,7 @@ use std::{
 
 use chrono::{NaiveDateTime, Utc};
 use futures::{stream::FuturesUnordered, StreamExt};
-use serde::de::DeserializeOwned;
+use serde::{de::DeserializeOwned, Deserialize};
 use sqlx::{Pool, Postgres};
 use tokio::{
     join,
@@ -30,6 +30,8 @@ use windmill_api::{
 
 #[cfg(feature = "enterprise")]
 use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
+#[cfg(feature = "enterprise")]
+use windmill_common::ee::low_disk_alerts;
 
 #[cfg(feature = "oauth2")]
 use windmill_common::global_settings::OAUTH_SETTING;
@@ -40,13 +42,14 @@ use windmill_common::{
     error,
     flow_status::{FlowStatus, FlowStatusModule},
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
-        DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
+        CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
+        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
+        HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
+        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
+        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING,
+        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -65,10 +68,10 @@ use windmill_common::{
         DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR,
         WORKER_CONFIG, WORKER_GROUP,
     },
-    KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB,
-    DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
-    MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
-    SERVICE_LOG_RETENTION_SECS,
+    KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED,
+    CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
+    METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED,
+    OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS,
 };
 use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
 use windmill_worker::{
@@ -134,7 +137,7 @@ lazy_static::lazy_static! {
     .and_then(|x| x.parse::<bool>().ok())
     .unwrap_or(false);
 
-
+    pub static ref WORKERS_NAMES: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
 
     static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
     static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
@@ -183,6 +186,12 @@ pub async fn initial_load(
     if server_mode {
         if let Some(db) = conn.as_sql() {
             load_require_preexisting_user(db).await;
+            if let Err(e) = reload_critical_alerts_on_db_oversize(db).await {
+                tracing::error!(
+                    "Error reloading critical alerts on db oversize setting: {:?}",
+                    e
+                )
+            }
         }
     }
 
@@ -1406,6 +1415,23 @@ pub async fn monitor_db(
         }
     };
 
+    let low_disk_alerts_f = async {
+        #[cfg(feature = "enterprise")]
+        if let Some(db) = conn.as_sql() {
+            low_disk_alerts(
+                &db,
+                server_mode,
+                _worker_mode,
+                WORKERS_NAMES.read().await.clone(),
+            )
+            .await;
+        }
+        #[cfg(not(feature = "enterprise"))]
+        {
+            ()
+        }
+    };
+
     let apply_autoscaling_f = async {
         #[cfg(feature = "enterprise")]
         if server_mode && !initial_load {
@@ -1428,6 +1454,7 @@ pub async fn monitor_db(
         verify_license_key_f,
         worker_groups_alerts_f,
         jobs_waiting_alerts_f,
+        low_disk_alerts_f,
         apply_autoscaling_f,
         update_min_worker_version_f,
     );
@@ -2183,6 +2210,37 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<
     Ok(())
 }
 
+pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<()> {
+    #[derive(Deserialize)]
+    struct DBOversize {
+        enabled: bool,
+        value: f32,
+    }
+    let db_oversize_value =
+        load_value_from_global_settings(conn, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING).await?;
+
+    let db_oversize = if let Some(q) = db_oversize_value {
+        match serde_json::from_value::<DBOversize>(q.clone()) {
+            Ok(DBOversize { enabled: true, value }) => Some(value),
+            Ok(_) => None,
+            Err(q) => {
+                tracing::error!(
+                    "Could not parse critical_alerts_on_db_oversize setting, found: {:#?}",
+                    &q
+                );
+                None
+            }
+        }
+    } else {
+        None
+    };
+
+    let mut l = CRITICAL_ALERTS_ON_DB_OVERSIZE.write().await;
+    *l = db_oversize;
+
+    Ok(())
+}
+
 async fn generate_and_save_jwt_secret(db: &DB) -> error::Result<String> {
     let secret = rd_string(32);
     sqlx::query!(

backend/tests/worker.rs

@@ -23,9 +23,9 @@ use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
 use windmill_api_client::types::{NewScript, ScriptLang as NewScriptLanguage};
 
 use serde::Serialize;
-use windmill_common::worker::WORKER_CONFIG;
 #[cfg(feature = "deno_core")]
 use windmill_common::flows::InputTransform;
+use windmill_common::worker::WORKER_CONFIG;
 
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule, RestartedFrom},
@@ -145,7 +145,7 @@ impl ApiServer {
             format!("http://localhost:{}", addr.port()),
         ));
 
-        _port_rx.await.unwrap();
+        _port_rx.await.expect("failed to receive port");
 
         // clear the cache between tests
         windmill_common::cache::clear();
@@ -171,7 +171,7 @@ impl ApiServer {
 //     Ok(())
 // }
 
-#[cfg(feature="python")]
+#[cfg(feature = "python")]
 fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     cjob.flow_status.clone().and_then(|fs| {
         find_module_in_vec(
@@ -181,7 +181,7 @@ fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
     })
 }
 
-#[cfg(feature="python")]
+#[cfg(feature = "python")]
 fn find_module_in_vec(modules: Vec<FlowStatusModule>, id: &str) -> Option<FlowStatusModule> {
     modules.into_iter().find(|s| s.id() == id)
 }

backend/windmill-api/Cargo.toml

@@ -15,7 +15,7 @@ stripe = []
 agent_worker_server = []
 enterprise_saml = ["dep:samael", "dep:libxml"]
 benchmark = []
-embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"]
+embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
 parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
 openidconnect = ["dep:openidconnect"]
@@ -54,7 +54,6 @@ tokio-stream.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
 axum.workspace = true
-half = { workspace = true, optional = true}
 futures.workspace = true
 git-version.workspace = true
 tower.workspace = true

backend/windmill-api/src/embeddings.rs

@@ -253,7 +253,7 @@ impl ModelInstance {
             let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?;
             let token_type_ids = token_ids.zeros_like()?;
 
-            let embedding = self.model.forward(&token_ids, &token_type_ids)?;
+            let embedding = self.model.forward(&token_ids, &token_type_ids, None)?;
             let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?;
             let embedding = normalize_l2(&embedding)?;
 

backend/windmill-api/src/scripts.rs

@@ -265,9 +265,12 @@ async fn list_scripts(
 
     if lq.show_archived.unwrap_or(false) {
         sqlb.and_where_eq(
-            "o.created_at",
-            "(select max(created_at) from script where o.path = path 
-            AND workspace_id = ?)"
+            "o.ctid",
+            "(SELECT ctid FROM script 
+              WHERE path = o.path 
+                AND workspace_id = ? 
+              ORDER BY created_at DESC 
+              LIMIT 1)"
                 .bind(&w_id),
         );
         sqlb.and_where_eq("archived", true);
@@ -989,7 +992,7 @@ async fn get_script_by_path(
                 AND favorite.usr = $3
             WHERE s.path = $1
                 AND s.workspace_id = $2
-                AND s.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)",
+            ORDER BY s.created_at DESC LIMIT 1",
         )
         .bind(path)
         .bind(w_id)
@@ -998,9 +1001,7 @@ async fn get_script_by_path(
         .await?
     } else {
         sqlx::query_as::<_, ScriptWithStarred>(
-            "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 \
-             AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
-             workspace_id = $2)",
+            "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1",
         )
         .bind(path)
         .bind(w_id)
@@ -1040,9 +1041,8 @@ async fn get_script_by_path_w_draft(
     let script_o = sqlx::query_as::<_, ScriptWDraft>(
         "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON 
          script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
-         WHERE script.path = $1 AND script.workspace_id = $2 \
-         AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
-         workspace_id = $2)",
+         WHERE script.path = $1 AND script.workspace_id = $2
+         ORDER BY script.created_at DESC LIMIT 1",
     )
     .bind(path)
     .bind(w_id)
@@ -1064,7 +1064,7 @@ async fn get_script_history(
         "SELECT s.hash as hash, dm.deployment_msg as deployment_msg 
         FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash
         WHERE s.workspace_id = $1 AND s.path = $2
-        ORDER by created_at DESC",
+        ORDER by s.created_at DESC",
         w_id,
         path.to_path(),
     )
@@ -1092,7 +1092,7 @@ async fn get_latest_version(
         "SELECT s.hash as hash, dm.deployment_msg as deployment_msg 
         FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash
         WHERE s.workspace_id = $1 AND s.path = $2
-        ORDER by created_at DESC",
+        ORDER by s.created_at DESC LIMIT 1",
         w_id,
         path.to_path(),
     )
@@ -1188,7 +1188,15 @@ async fn toggle_workspace_error_handler(
     match error_handler_maybe {
         Some(_) => {
             sqlx::query_scalar!(
-                "UPDATE script SET ws_error_handler_muted = $3 WHERE workspace_id = $2 AND path = $1 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)",
+                "UPDATE script 
+                SET ws_error_handler_muted = $3 
+                WHERE ctid = (
+                    SELECT ctid FROM script
+                    WHERE path = $1 AND workspace_id = $2
+                    ORDER BY created_at DESC
+                    LIMIT 1
+                )
+",
                 path.to_path(),
                 w_id,
                 req.muted,
@@ -1209,6 +1217,7 @@ async fn toggle_workspace_error_handler(
 
 async fn get_tokened_raw_script_by_path(
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, token, path)): Path<(String, String, StripPath)>,
     Extension(cache): Extension<Arc<AuthCache>>,
 ) -> Result<String> {
@@ -1216,7 +1225,13 @@ async fn get_tokened_raw_script_by_path(
         .get_authed(Some(w_id.clone()), &token)
         .await
         .ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?;
-    return raw_script_by_path(authed, Extension(user_db), Path((w_id, path))).await;
+    return raw_script_by_path(
+        authed,
+        Extension(user_db),
+        Extension(db),
+        Path((w_id, path)),
+    )
+    .await;
 }
 
 async fn get_empty_ts_script_by_path() -> String {
@@ -1226,22 +1241,25 @@ async fn get_empty_ts_script_by_path() -> String {
 async fn raw_script_by_path(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
-    raw_script_by_path_internal(path, user_db, authed, w_id, false).await
+    raw_script_by_path_internal(path, user_db, db, authed, w_id, false).await
 }
 
 async fn raw_script_by_path_unpinned(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
-    raw_script_by_path_internal(path, user_db, authed, w_id, true).await
+    raw_script_by_path_internal(path, user_db, db, authed, w_id, true).await
 }
 
 async fn raw_script_by_path_internal(
     path: StripPath,
     user_db: UserDB,
+    db: DB,
     authed: ApiAuthed,
     w_id: String,
     unpin: bool,
@@ -1267,10 +1285,7 @@ async fn raw_script_by_path_internal(
     let mut tx = user_db.begin(&authed).await?;
 
     let content_o = sqlx::query_scalar!(
-        "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 \
-         AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND \
-         workspace_id = $2)",
+        "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1",
         path,
         w_id
     )
@@ -1278,6 +1293,22 @@ async fn raw_script_by_path_internal(
     .await?;
     tx.commit().await?;
 
+    if content_o.is_none() {
+        let exists = sqlx::query_scalar!(
+            "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
+            path,
+            w_id
+        )
+        .fetch_one(&db)
+        .await?;
+        if exists.unwrap_or(false) {
+            return Err(Error::NotFound(format!(
+                "Script {path} not visible to {} but exists",
+                authed.username
+            )));
+        }
+    }
+
     let content = not_found_if_none(content_o, "Script", path)?;
 
     if unpin {
@@ -1294,8 +1325,7 @@ async fn exists_script_by_path(
     let path = path.to_path();
 
     let exists = sqlx::query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND
-         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2))",
+        "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
         path,
         w_id
     )
@@ -1412,9 +1442,7 @@ pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: D
         path,
         w_id,
         db,
-        "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 \
-             AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
-             workspace_id = $2)",
+        "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1",
         "script",
     )
     .await;

backend/windmill-common/Cargo.toml

@@ -76,6 +76,8 @@ quick_cache.workspace = true
 pin-project-lite.workspace = true
 futures.workspace = true
 tempfile.workspace = true
+systemstat.workspace = true
+size.workspace = true
 
 opentelemetry-semantic-conventions = { workspace = true, optional = true }
 opentelemetry-otlp = { workspace = true, optional = true }

backend/windmill-common/src/ee.rs

@@ -98,3 +98,9 @@ pub async fn worker_groups_alerts(_db: &DB) {}
 
 #[cfg(feature = "enterprise")]
 pub async fn jobs_waiting_alerts(_db: &DB) {}
+
+#[cfg(feature = "enterprise")]
+pub async fn low_disk_alerts(_db: &DB, _server_mode: bool, _worker_mode: bool, _workers: Vec<String>) {
+    // Implementation is not open source
+}
+

backend/windmill-common/src/flows.rs

@@ -22,7 +22,8 @@ use crate::{
     error::Error,
     more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
     scripts::{Schema, ScriptHash, ScriptLang},
-    worker::{to_raw_value, Connection}, DB,
+    worker::{to_raw_value, Connection},
+    DB,
 };
 
 #[derive(Serialize, Deserialize, sqlx::FromRow)]

backend/windmill-common/src/global_settings.rs

@@ -37,6 +37,7 @@ pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
 pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url";
 pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels";
 pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
+pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize";
 pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
 pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";

backend/windmill-common/src/lib.rs

@@ -115,6 +115,7 @@ lazy_static::lazy_static! {
 
 
     pub static ref CRITICAL_ERROR_CHANNELS: Arc<RwLock<Vec<CriticalErrorChannel>>> = Arc::new(RwLock::new(vec![]));
+    pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: Arc<RwLock<Option<f32>>> = Arc::new(RwLock::new(None));
 
     pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
 

backend/windmill-worker/src/ansible_executor.rs

@@ -858,18 +858,16 @@ pub async fn handle_ansible_job(
 
     let mut nsjail_extra_mounts = vec![];
     if let Some(r) = reqs.as_ref() {
-        if let Some(db) = conn.as_sql() {
-            nsjail_extra_mounts = create_file_resources(
-                &job.id,
-                &job.workspace_id,
-                job_dir,
-                interpolated_args.as_ref(),
-                &r,
-                &client,
-                db,
-            )
-            .await?;
-        }
+        nsjail_extra_mounts = create_file_resources(
+            &job.id,
+            &job.workspace_id,
+            job_dir,
+            interpolated_args.as_ref(),
+            &r,
+            &client,
+            conn,
+        )
+        .await?;
 
         for repo in &r.git_repos {
             append_logs(
@@ -935,8 +933,7 @@ pub async fn handle_ansible_job(
 
         if let Some(collections) = r.roles_and_collections.as_ref() {
             let empty = String::new();
-            let (lockfile, logs) =
-                req_lockfiles
+            let (lockfile, logs) = req_lockfiles
                 .as_ref()
                 .map(|r| (&r.collections_and_roles, &r.collections_and_roles_logs))
                 .unwrap_or((collections, &empty));
@@ -1162,7 +1159,7 @@ async fn create_file_resources(
     args: Option<&HashMap<String, Box<RawValue>>>,
     r: &AnsibleRequirements,
     client: &crate::AuthedClient,
-    db: &sqlx::Pool<sqlx::Postgres>,
+    conn: &Connection,
 ) -> error::Result<Vec<String>> {
     let mut logs = String::new();
     let mut nsjail_mounts: Vec<String> = vec![];
@@ -1232,7 +1229,7 @@ async fn create_file_resources(
             file_res.target_path, file_res.resource_path
         ));
     }
-    append_logs(job_id, w_id, logs, &Connection::Sql(db.clone())).await;
+    append_logs(job_id, w_id, logs, conn).await;
 
     Ok(nsjail_mounts)
 }

flake.nix

@@ -17,6 +17,7 @@
           extensions = [
             "rust-src" # for rust-analyzer
             "rust-analyzer"
+            "rustfmt"
           ];
         };
         buildInputs = with pkgs; [
@@ -183,6 +184,7 @@
           ANSIBLE_PLAYBOOK_PATH = "${pkgs.ansible}/bin/ansible-playbook";
           ANSIBLE_GALAXY_PATH = "${pkgs.ansible}/bin/ansible-galaxy";
           RUST_LOG = "debug";
+          SQLX_OFFLINE = "true";
         };
         packages.default = self.packages.${system}.windmill;
         packages.windmill-client = pkgs.buildNpmPackage {

frontend/src/lib/components/InstanceSetting.svelte

@@ -334,15 +334,15 @@
 												latestKeyRenewalAttempt.result === 'success'
 													? 'text-green-600'
 													: isTrial
-													? 'text-yellow-600'
-													: 'text-red-600'
+														? 'text-yellow-600'
+														: 'text-red-600'
 											)}
 										>
 											{latestKeyRenewalAttempt.result === 'success'
 												? 'Latest key renewal succeeded'
 												: isTrial
-												? 'Latest key renewal ignored because in trial'
-												: 'Latest key renewal failed'}
+													? 'Latest key renewal ignored because in trial'
+													: 'Latest key renewal failed'}
 											on {attemptedAt}
 										</span>
 									</div>
@@ -928,6 +928,30 @@
 				{:else if setting.fieldType == 'object_store_config'}
 					<ObjectStoreConfigSettings bind:bucket_config={$values[setting.key]} />
 					<div class="mb-6"></div>
+				{:else if setting.fieldType == 'critical_alerts_on_db_oversize'}
+					{#if $values[setting.key]}
+						<div class="flex flex-row flex-wrap gap-2 p-0 items-center">
+							<div class="p-1">
+								<Toggle
+									disabled={!$enterpriseLicense}
+									bind:checked={$values[setting.key].enabled}
+								/>
+							</div>
+							{#if $values[setting.key].enabled}
+								<label class="block shrink min-w-0">
+									<input
+										type="number"
+										placeholder={setting.placeholder}
+										bind:value={$values[setting.key].value}
+									/>
+								</label>
+								<label class="block">
+									<span class="text-primary font-semibold text-sm">GB</span>
+								</label>
+							{/if}
+						</div>
+						<div class="mb-6"></div>
+					{/if}
 				{:else if setting.fieldType == 'number'}
 					<input
 						type="number"

frontend/src/lib/components/instanceSettings.ts

@@ -26,6 +26,7 @@ export interface Setting {
 		| 'license_key'
 		| 'object_store_config'
 		| 'critical_error_channels'
+		| 'critical_alerts_on_db_oversize'
 		| 'slack_connect'
 		| 'smtp_connect'
 		| 'indexer_rates'
@@ -353,6 +354,15 @@ export const settings: Record<string, Setting[]> = {
 			fieldType: 'smtp_connect',
 			storage: 'setting',
 			ee_only: ''
+		},
+		{
+			label: 'Alert on DB oversize',
+			key: 'critical_alerts_on_db_oversize',
+			description: 'Alert if DB grows more than specified size',
+			fieldType: 'critical_alerts_on_db_oversize',
+			placeholder: '100',
+			storage: 'setting',
+			ee_only: ''
 		}
 	],
 	'OTEL/Prom': [