diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e0d990384a..a71cab586d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9b3339730eb4bb0b564c7c56ac546f33fb3d8905 \ No newline at end of file +6fd5a2ce908235a17975ad4dbdf0051cd89334f3 diff --git a/backend/src/main.rs b/backend/src/main.rs index c4440b7f34..97980484a5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -62,7 +62,7 @@ use windmill_common::{ }, worker::{ is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR, - HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, + HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP, }, KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED, }; @@ -238,8 +238,8 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { ) })?; - create_dir_all(HUB_CACHE_DIR)?; - create_dir_all(BUN_BUNDLE_CACHE_DIR)?; + create_dir_all(&*HUB_CACHE_DIR)?; + create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?; for path in paths.values() { tracing::info!("Caching hub script at {path}"); @@ -249,7 +249,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { .as_ref() .is_some_and(|x| x == &ScriptLang::Deno) { - let job_dir = format!("{}/cache_init/{}", TMP_DIR, Uuid::new_v4()); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, Uuid::new_v4()); create_dir_all(&job_dir)?; let _ = windmill_worker::generate_deno_lock( &Uuid::nil(), @@ -267,7 +267,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { tokio::fs::remove_dir_all(job_dir).await?; } else if res.language.as_ref().is_some_and(|x| x == &ScriptLang::Bun) { let job_id = Uuid::new_v4(); - let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id); + let job_dir = format!("{}/cache_init/{}", *WINDMILL_DIR, job_id); create_dir_all(&job_dir)?; if let Some(lock) = res.lockfile { let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?; @@ -384,9 +384,9 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Fetched {} resource types from hub", resource_types.len()); - create_dir_all(HUB_RT_CACHE_DIR)?; + create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -398,7 +398,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { } pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); if tokio::fs::metadata(&cache_path).await.is_err() { tracing::info!( @@ -969,7 +969,7 @@ Windmill Community Edition {GIT_VERSION} DirBuilder::new() .recursive(true) - .create("/tmp/windmill") + .create(&*WINDMILL_DIR) .expect("could not create initial server dir"); #[cfg(feature = "tantivy")] @@ -1794,27 +1794,27 @@ pub async fn run_workers( let mut handles = Vec::with_capacity(num_workers as usize); for x in [ - TMP_LOGS_DIR, - UV_CACHE_DIR, - DENO_CACHE_DIR, - DENO_CACHE_DIR_DEPS, - DENO_CACHE_DIR_NPM, - BUN_CACHE_DIR, - PY310_CACHE_DIR, - PY311_CACHE_DIR, - PY312_CACHE_DIR, - PY313_CACHE_DIR, - BUN_BUNDLE_CACHE_DIR, - GO_CACHE_DIR, - GO_BIN_CACHE_DIR, - RUST_CACHE_DIR, - CSHARP_CACHE_DIR, - NU_CACHE_DIR, - HUB_CACHE_DIR, - POWERSHELL_CACHE_DIR, - JAVA_CACHE_DIR, - RUBY_CACHE_DIR, - TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG + &*TMP_LOGS_DIR, + &*UV_CACHE_DIR, + &*DENO_CACHE_DIR, + &*DENO_CACHE_DIR_DEPS, + &*DENO_CACHE_DIR_NPM, + &*BUN_CACHE_DIR, + &*PY310_CACHE_DIR, + &*PY311_CACHE_DIR, + &*PY312_CACHE_DIR, + &*PY313_CACHE_DIR, + &*BUN_BUNDLE_CACHE_DIR, + &*GO_CACHE_DIR, + &*GO_BIN_CACHE_DIR, + &*RUST_CACHE_DIR, + &*CSHARP_CACHE_DIR, + &*NU_CACHE_DIR, + &*HUB_CACHE_DIR, + &*POWERSHELL_CACHE_DIR, + &*JAVA_CACHE_DIR, + &*RUBY_CACHE_DIR, + &*TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG ] { DirBuilder::new() .recursive(true) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 17e79c3393..6baba34a02 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -73,7 +73,7 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, - DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, + DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, @@ -595,7 +595,7 @@ async fn sleep_until_next_minute_start_plus_one_s() { use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; async fn find_two_highest_files(hostname: &str) -> (Option, Option) { - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); let rd_dir = tokio::fs::read_dir(log_dir).await; if let Ok(mut log_files) = rd_dir { let mut highest_file: Option = None; @@ -614,7 +614,8 @@ async fn find_two_highest_files(hostname: &str) -> (Option, Option () { .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; + delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; } Err(e) => tracing::error!("Error deleting log file: {:?}", e), @@ -1140,7 +1141,7 @@ async fn delete_expired_jobs_batch( .filter_map(|opt| opt) .flat_map(|inner_vec| inner_vec.into_iter()) .collect(); - delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await; + delete_log_files_from_disk_and_store(paths, &*WINDMILL_DIR, "").await; } Err(e) => tracing::error!("Error deleting job logs: {:?}", e), } @@ -1367,7 +1368,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { let settings_xml = MAVEN_SETTINGS_XML.read().await.clone(); match settings_xml { Some(ref content) if !content.trim().is_empty() => { - let m2_dir = format!("{JAVA_HOME_DIR}/.m2"); + let m2_dir = format!("{}/.m2", *JAVA_HOME_DIR); if let Err(e) = tokio::fs::create_dir_all(&m2_dir).await { tracing::error!("Failed to create .m2 directory: {e:#}"); return; @@ -1378,7 +1379,7 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { } } _ => { - let settings_path = format!("{JAVA_HOME_DIR}/.m2/settings.xml"); + let settings_path = format!("{}/.m2/settings.xml", *JAVA_HOME_DIR); let _ = tokio::fs::remove_file(&settings_path).await; } } diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 9717403830..082a43be2c 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -206,7 +206,7 @@ fn spawn_workers( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, _) = KillpillSender::new(n + 1); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index dd494de007..22c2313b91 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1,7 +1,7 @@ -use windmill_test_utils::*; use sqlx::postgres::Postgres; use sqlx::Pool; use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] @@ -188,7 +188,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -207,14 +208,14 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_global_site_packages(db: Pool) -> anyhow::Result<()> { - use windmill_common::{cache::concatcp, worker::ROOT_CACHE_DIR}; + use windmill_common::worker::ROOT_CACHE_DIR; initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); // Shared for all 3.12.* - let path = concatcp!(ROOT_CACHE_DIR, "python_3_12/global-site-packages").to_owned(); + let path = format!("{}python_3_12/global-site-packages", *ROOT_CACHE_DIR); std::fs::create_dir_all(&path).unwrap(); std::fs::write(path + "/my_global_site_package_3_12_any.py", "").unwrap(); @@ -237,7 +238,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -271,7 +274,9 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -310,7 +315,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, @@ -347,7 +353,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 04a1eb617b..bf493f9420 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -1085,7 +1085,7 @@ async fn sync_cached_resource_types( require_super_admin(&db, &authed.email).await?; use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", HUB_RT_CACHE_DIR); + let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); let content = tokio::fs::read_to_string(&cache_path).await.map_err(|e| { error::Error::NotFound(format!( diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 420bbf24b9..d58dcd3c69 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -47,7 +47,7 @@ use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAsset use windmill_common::scripts::ScriptRunnableSettingsInline; use windmill_common::triggers::TriggerMetadata; use windmill_common::utils::{RunnableKind, WarnAfterExt}; -use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; +use windmill_common::worker::{Connection, CLOUD_HOSTED, WINDMILL_DIR}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; @@ -1412,7 +1412,7 @@ async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -1427,7 +1427,7 @@ async fn get_logs_from_disk( "#.to_string(), )); for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; @@ -5888,7 +5888,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R )); } - let local_file = format!("{TMP_DIR}/logs/{file_p}"); + let local_file = format!("{}/logs/{file_p}", *WINDMILL_DIR); if tokio::fs::metadata(&local_file).await.is_ok() { let mut file = tokio::fs::File::open(local_file).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); @@ -5934,10 +5934,10 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - return Err(error::Error::NotFound(format!( - "File not found on server logs volume /tmp/windmill/logs and no distributed logs s3 storage for {}", - file_p - ))); + return Err(error::Error::NotFound(format!( + "File not found on server logs volume {}/logs and no distributed logs s3 storage for {}", + *WINDMILL_DIR, file_p + ))); } async fn get_job_update( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 9b713cad0a..528e60b02f 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -262,7 +262,7 @@ pub async fn run_server( ) -> anyhow::Result<()> { let user_db = UserDB::new(db.clone()); - for x in [HUB_CACHE_DIR] { + for x in [&*HUB_CACHE_DIR] { DirBuilder::new() .recursive(true) .create(x) diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 20d2b58d0e..c83bb21f2c 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -102,7 +102,11 @@ async fn get_log_file( #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client.get(&windmill_object_store::object_store_reexports::Path::from(path)).await; + let file = s3_client + .get(&windmill_object_store::object_store_reexports::Path::from( + path, + )) + .await; match file { Ok(file) => { let bytes = file.bytes().await; @@ -126,7 +130,7 @@ async fn get_log_file( } } } - let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await; + let file = tokio::fs::read(format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path)).await; if let Ok(bytes) = file { Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))) } else { diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c881e9ae9b..091c465a61 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -43,6 +43,7 @@ use windmill_common::runnable_settings::{ConcurrencySettings, DebouncingSettings use windmill_common::scripts::ScriptRunnableSettingsHandle; use windmill_common::utils::require_admin; use windmill_common::variables::decrypt; +use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, @@ -372,7 +373,7 @@ pub(crate) async fn tarball_workspace( let mut tx = user_db.begin(&authed).await?; - let tmp_dir = TempDir::new_in("/tmp/windmill/")?; + let tmp_dir = TempDir::new_in(&*WINDMILL_DIR)?; let name = match archive_type.as_deref() { Some("tar") | None => Ok(format!("windmill-{w_id}.tar")), diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-common/src/bench.rs index b87872119b..1ed2b9aa56 100644 --- a/backend/windmill-common/src/bench.rs +++ b/backend/windmill-common/src/bench.rs @@ -1,5 +1,5 @@ use crate::{ - worker::{write_file, TMP_DIR}, + worker::{write_file, WINDMILL_DIR}, DB, }; use serde::Serialize; @@ -113,7 +113,8 @@ impl BenchmarkInfo { "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}{pool_info}", self.iters as f64 / total_duration as f64 * 1000.0 ); - write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); + write_file(&WINDMILL_DIR, path, &serde_json::to_string(&self).unwrap()) + .expect("write profiling"); Ok(()) } } diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index bc40e4cd75..583914c573 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -18,7 +18,7 @@ use crate::{ scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, - worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, TMP_DIR}, + worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -225,7 +225,7 @@ pub async fn get_logs_from_disk( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { for file_p in &file_index { - if !tokio::fs::metadata(format!("{TMP_DIR}/{file_p}")) + if !tokio::fs::metadata(format!("{}/{file_p}", *WINDMILL_DIR)) .await .is_ok() { @@ -236,7 +236,7 @@ pub async fn get_logs_from_disk( let logs = logs.to_string(); let stream = async_stream::stream! { for file_p in file_index.clone() { - let mut file = tokio::fs::File::open(format!("{TMP_DIR}/{file_p}")).await.map_err(to_anyhow)?; + let mut file = tokio::fs::File::open(format!("{}/{file_p}", *WINDMILL_DIR)).await.map_err(to_anyhow)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer).await.map_err(to_anyhow)?; yield Ok(bytes::Bytes::from(buffer)) as anyhow::Result; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 68f77a2303..cfff28dbcb 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -206,12 +206,12 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; - let cache_path = format!("{HUB_CACHE_DIR}/{version}"); + let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); let script; if tokio::fs::metadata(&cache_path).await.is_err() { script = get_full_hub_script_by_path_inner(path, http_client, db).await?; if let Err(e) = crate::worker::write_file( - HUB_CACHE_DIR, + &HUB_CACHE_DIR, &version, &serde_json::to_string(&script).map_err(to_anyhow)?, ) { diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 701a0887d0..f4892b67c9 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -6,8 +6,6 @@ * LICENSE-AGPL for a copy of the license. */ -use const_format::concatcp; - use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -61,7 +59,9 @@ fn create_targets_filter(default_env_filter: LevelFilter) -> Targets { pub const LOGS_SERVICE: &str = "logs/services/"; -pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); +lazy_static::lazy_static! { + pub static ref TMP_WINDMILL_LOGS_SERVICE: String = format!("{}/{}", *crate::worker::WINDMILL_DIR, LOGS_SERVICE); +} pub fn initialize_tracing( hostname: &str, @@ -108,7 +108,7 @@ pub fn initialize_tracing( use tracing_appender::rolling::{RollingFileAppender, Rotation}; - let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); std::fs::create_dir_all(&log_dir).unwrap(); let file_appender = RollingFileAppender::builder() .rotation(Rotation::MINUTELY) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 1a2d55b896..8d95541f61 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -2,7 +2,6 @@ use anyhow::anyhow; use axum::http::HeaderMap; use bytes::Bytes; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use reqwest_middleware::ClientWithMiddleware; @@ -274,7 +273,9 @@ lazy_static::lazy_static! { pub static ref ROOT_STANDALONE_BUNDLE_DIR: String = format!("{}/.windmill/standalone_bundle", std::env::var("HOME").unwrap_or_else(|_| "/root".to_string())); } -pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); +lazy_static::lazy_static! { + pub static ref ROOT_CACHE_NOMOUNT_DIR: String = format!("{}/cache_nomount/", *WINDMILL_DIR); +} /// Whether native mode is forced by the environment (NATIVE_MODE=true env var or WORKER_GROUP=native). /// This does NOT account for native_mode set in the DB worker group config — for that, read @@ -490,13 +491,23 @@ pub async fn store_pull_query(wc: &WorkerConfig) { *l = queries; } -pub const TMP_DIR: &str = "/tmp/windmill"; -pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); - -pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); -pub const HUB_RT_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub_rt"); - -pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/"); +lazy_static::lazy_static! { + pub static ref WINDMILL_DIR: String = { + let dir = std::env::var("WINDMILL_DIR") + .unwrap_or_else(|_| "/tmp/windmill".to_string()); + if dir.is_empty() { + panic!("WINDMILL_DIR must not be empty"); + } + if dir.ends_with('/') || dir.ends_with('\\') { + panic!("WINDMILL_DIR must not end with a trailing slash, got: {dir}"); + } + dir + }; + pub static ref TMP_LOGS_DIR: String = format!("{}/logs", *WINDMILL_DIR); + pub static ref ROOT_CACHE_DIR: String = format!("{}/cache/", *WINDMILL_DIR); + pub static ref HUB_CACHE_DIR: String = format!("{}hub", *ROOT_CACHE_DIR); + pub static ref HUB_RT_CACHE_DIR: String = format!("{}hub_rt", *ROOT_CACHE_DIR); +} pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { let path = format!("{}/{}", dir, path); diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index a523106cbc..e81925e7bb 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -45,7 +45,7 @@ use uuid::Uuid; use windmill_common::error::Error; use windmill_common::result_stream::append_result_stream_db; -use windmill_common::worker::{write_file, Connection, TMP_DIR}; +use windmill_common::worker::{write_file, Connection, WINDMILL_DIR}; // ── Permission container ───────────────────────────────────────────── @@ -151,7 +151,9 @@ static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH pub(crate) const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); -const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); +lazy_static::lazy_static! { + static ref ERROR_DIR: String = format!("{}/native_errors", *WINDMILL_DIR); +} lazy_static! { static ref RE_PROXY: Regex = @@ -263,14 +265,14 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { } fn write_error_expr(expr: &str, uuid: &Uuid) { - if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { - tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + if let Err(e) = std::fs::create_dir_all(&*ERROR_DIR) { + tracing::error!("failed to create error dir {}: {e}", *ERROR_DIR); return; } - let dir_entries = match std::fs::read_dir(ERROR_DIR) { + let dir_entries = match std::fs::read_dir(&*ERROR_DIR) { Ok(entries) => entries.count(), Err(_) => { - tracing::error!("failed to read error dir {ERROR_DIR}"); + tracing::error!("failed to read error dir {}", *ERROR_DIR); return; } }; @@ -279,15 +281,16 @@ fn write_error_expr(expr: &str, uuid: &Uuid) { tracing::info!("native error for job {uuid}: {expr}"); } if dir_entries >= 100 { - tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + tracing::info!("Too many error files in {}, skipping write", *ERROR_DIR); return; } let path = format!("/{uuid}.js"); tracing::info!( - "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + "nativets job {uuid} failed, writing error expr to {}/{path} for debugging: {path}", + *ERROR_DIR ); - if let Err(e) = write_file(ERROR_DIR, &path, expr) { + if let Err(e) = write_file(&ERROR_DIR, &path, expr) { tracing::error!("failed to write error expr to file {path}: {e}"); } } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index bcac53e7d2..6b414ca10f 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -47,7 +47,7 @@ use windmill_common::{ StripPath, }, variables, - worker::{CLOUD_HOSTED, TMP_DIR}, + worker::{CLOUD_HOSTED, WINDMILL_DIR}, PgDatabase, }; @@ -1752,7 +1752,7 @@ async fn write_ssh_file( var_path: &str, ) -> std::result::Result { let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); - let loc = std::path::Path::new(TMP_DIR) + let loc = std::path::Path::new(&*WINDMILL_DIR) .join("ssh_ids") .join(id_file_name); diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index adbeb83ee7..69e7fc0558 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -380,7 +380,7 @@ pub fn spawn_test_worker( std::fs::DirBuilder::new() .recursive(true) - .create(windmill_worker::GO_BIN_CACHE_DIR) + .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, rx) = KillpillSender::new(1); diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index 87b6abda21..afe9d5df9f 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -127,7 +127,7 @@ iface_no_lo: true mount { src: "{CACHE_DIR}" - dst: "/tmp/windmill/cache/powershell" + dst: "{CACHE_DIR}" is_bind: true rw: false mandatory: false diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index aa250de4ab..253384f6f2 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,9 +33,9 @@ use crate::{ read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, + is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - is_sandboxing_enabled, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1184,7 +1184,7 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 64270b4044..30348895cf 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -194,7 +194,7 @@ exit $exit_status .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut cmd_args = vec![ diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 9c46db572d..08a61645ce 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -737,7 +737,7 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result Error { pub async fn clean_cache() -> error::Result<()> { tracing::info!("Started cleaning cache"); - tokio::fs::remove_dir_all(ROOT_CACHE_DIR).await?; + tokio::fs::remove_dir_all(&*ROOT_CACHE_DIR).await?; tracing::info!("Finished cleaning cache"); Ok(()) } @@ -1557,4 +1557,3 @@ mod tests { assert!(result.is_err()); } } - diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index e4c32648d0..4c3e63f4c9 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,14 +13,11 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{ - utils::calculate_hash, - worker::write_file, -}; +use windmill_common::{utils::calculate_hash, worker::write_file}; -use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; +use windmill_common::error::{self, Error}; #[cfg(feature = "csharp")] use windmill_queue::append_logs; @@ -105,8 +102,8 @@ pub async fn generate_nuget_lockfile( let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -367,8 +364,8 @@ async fn build_cs_proj( .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("MSBUILDDISABLENODEREUSE", "1") @@ -434,7 +431,7 @@ async fn build_cs_proj( } } - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); #[cfg(unix)] let target = format!("{job_dir}/Main"); #[cfg(windows)] @@ -516,11 +513,10 @@ pub async fn handle_csharp_job( inner_content, requirements_o.unwrap_or(&String::new()) )); - let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] @@ -591,11 +587,11 @@ pub async fn handle_csharp_job( "run.config.proto", &NSJAIL_CONFIG_RUN_CSHARP_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", CSHARP_CACHE_DIR) + .replace("{CACHE_DIR}", &*CSHARP_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -608,8 +604,8 @@ pub async fn handle_csharp_job( .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) @@ -640,8 +636,8 @@ pub async fn handle_csharp_job( .envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("DOTNET_CLI_HOME", CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{CSHARP_CACHE_DIR}/nuget")) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") .env("DOTNET_NOLOGO", "true") .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 66d8ae8673..19831eadd0 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -443,7 +443,8 @@ try {{ } let allow_read = format!( - "--allow-read=./,/tmp/windmill/cache/deno/,{}", + "--allow-read=./,{}/,{}", + *DENO_CACHE_DIR, DENO_PATH.as_str() ); if let Some(deno_flags) = DENO_FLAGS.as_ref() { @@ -504,7 +505,8 @@ try {{ *has_stream = handle_result.result_stream.is_some(); // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); - if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await + if let Err(e) = + tokio::fs::remove_dir_all(format!("{}/gen/file/{job_dir}", *DENO_CACHE_DIR)).await { tracing::error!("failed to remove deno gen tmp cache dir: {}", e); } diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 812be2fc7a..ea5a9ded46 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -18,8 +18,8 @@ pub async fn build_tar_and_push( custom_folder_name: Option, platform_agnostic: bool, ) -> error::Result<()> { - use windmill_object_store::object_store_reexports::Path; use tokio::fs::create_dir_all; + use windmill_object_store::object_store_reexports::Path; use crate::TAR_PYBASE_CACHE_DIR; @@ -33,7 +33,7 @@ pub async fn build_tar_and_push( folder.split("/").last().unwrap().to_owned() }; - let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang); + let prefix = &format!("{}/{}", *TAR_PYBASE_CACHE_DIR, lang); let tar_path = format!("{prefix}/{folder_name}_tar.tar"); create_dir_all(prefix).await?; @@ -197,7 +197,9 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] if let Some(os) = windmill_object_store::get_object_store().await { return os - .get(&windmill_object_store::object_store_reexports::Path::from(_remote_path)) + .get(&windmill_object_store::object_store_reexports::Path::from( + _remote_path, + )) .await .is_ok(); } @@ -221,7 +223,7 @@ pub async fn save_cache( let file_to_cache = if is_dir { let tar_path = format!( "{}/tar/{}_tar.tar", - windmill_common::worker::ROOT_CACHE_DIR, + *windmill_common::worker::ROOT_CACHE_DIR, local_cache_path .split("/") .last() diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 35307baa72..29fff1a2ba 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -2,6 +2,7 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use windmill_common::scripts::ScriptLang; +use crate::global_cache::save_cache; use itertools::Itertools; use serde_json::value::RawValue; use tokio::{ @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection, GoAnnotations}, }; -use crate::global_cache::save_cache; use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -108,10 +108,9 @@ pub async fn handle_go_job( .expect("could not create go job dir"); let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock)); - let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); + let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) @@ -238,15 +237,15 @@ func Run(req Req) (interface{{}}, error){{ .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()) - .env("GOCACHE", GO_CACHE_DIR) + .env("GOCACHE", GO_CACHE_DIR.as_str()) .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) @@ -347,7 +346,7 @@ func Run(req Req) (interface{{}}, error){{ .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -384,11 +383,11 @@ func Run(req Req) (interface{{}}, error){{ .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .env("HOME", HOME_ENV.as_str()); @@ -508,7 +507,7 @@ pub async fn install_go_dependencies( #[cfg(windows)] child_cmd.env("GOPATH", windows_gopath()); #[cfg(unix)] - child_cmd.env("GOPATH", GO_CACHE_DIR); + child_cmd.env("GOPATH", GO_CACHE_DIR.as_str()); #[cfg(windows)] set_windows_env_vars(&mut child_cmd); @@ -591,11 +590,11 @@ pub async fn install_go_dependencies( .env("GOPATH", { #[cfg(unix)] { - GO_CACHE_DIR + GO_CACHE_DIR.as_str() } #[cfg(windows)] { - windows_gopath() + &windows_gopath() } }) .args(vec!["mod", mod_command]) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 49129848ce..efbe403145 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; +use crate::global_cache::save_cache; use anyhow::{anyhow, bail}; use async_recursion::async_recursion; use itertools::Itertools; @@ -15,7 +16,6 @@ use windmill_common::{ utils::calculate_hash, worker::{copy_dir_recursively, write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_parser::Arg; use windmill_parser_java::parse_java_sig_meta; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -185,8 +185,8 @@ pub async fn resolve<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies @@ -208,7 +208,7 @@ pub async fn resolve<'a>( cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -223,7 +223,7 @@ pub async fn resolve<'a>( "--parallel", &format!("{}", *JAVA_CONCURRENT_DOWNLOADS), "--cache", - COURSIER_CACHE_DIR, + &*COURSIER_CACHE_DIR, ]) .args(&get_repos(job_id, w_id, conn).await) .args(&deps.split("\n").collect_vec()) @@ -276,7 +276,8 @@ async fn install<'a>( match (it.next(), it.next(), it.next()) { (Some(group_id), Some(artifact_id), Some(version)) => { let path = format!( - "{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}", + "{}/{}/{artifact_id}/{version}", + *JAVA_REPOSITORY_DIR, group_id.replace(".", "/") ); Ok(RequiredDependency { @@ -312,7 +313,7 @@ async fn install<'a>( metadata(TRUST_STORE_PATH.clone()).await, ); let job_dir = job_dir.to_owned(); - let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4()); + let fetch_dir = format!("{}/tmp-fetch-{}", *JAVA_CACHE_DIR, Uuid::new_v4()); let fetch_dir2 = fetch_dir.clone(); par_install_language_dependencies_all_at_once( deps, @@ -334,8 +335,8 @@ async fn install<'a>( cmd.env_clear() .current_dir(&job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) - .env("COURSIER_CACHE", COURSIER_CACHE_DIR) + .env("HOME", &*JAVA_HOME_DIR) + .env("COURSIER_CACHE", &*COURSIER_CACHE_DIR) .envs(PROXY_ENVS.clone()); // Configure proxies { @@ -357,7 +358,7 @@ async fn install<'a>( } } - cmd.arg(&format!("-Duser.home={}", JAVA_HOME_DIR)); + cmd.arg(&format!("-Duser.home={}", *JAVA_HOME_DIR)); if trust_store_metadata.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), @@ -400,7 +401,7 @@ async fn install<'a>( if depth == 3 { copy_dir_recursively( &PathBuf::from(path), - &PathBuf::from(JAVA_REPOSITORY_DIR), + &PathBuf::from(&*JAVA_REPOSITORY_DIR), )?; return Ok(()); @@ -465,7 +466,7 @@ async fn compile<'a>( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let hash = compute_hash(inner_content, *requirements_o); - let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); + let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); let (cache, ..) = crate::global_cache::load_cache(&bin_path, &remote_path, true).await; @@ -501,7 +502,7 @@ async fn compile<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) @@ -604,7 +605,7 @@ async fn run<'a>( "run.config.proto", &NSJAIL_CONFIG_RUN_JAVA_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", JAVA_CACHE_DIR) + .replace("{CACHE_DIR}", &*JAVA_CACHE_DIR) .replace("{SHARED_MOUNT}", &shared_mount) // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), @@ -613,7 +614,7 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) @@ -671,7 +672,7 @@ async fn run<'a>( cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) - .env("HOME", JAVA_HOME_DIR) + .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables); diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index a4320a3612..ca309105b5 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,8 +16,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, - TRACING_PROXY_CA_CERT_PATH, + get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, + PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; use windmill_common::scripts::ScriptLang; @@ -253,7 +253,7 @@ async fn run<'a>( .replace("{NU_PATH}", &NU_PATH) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index acae81a588..961a49a58a 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -159,7 +159,7 @@ try { async fn scan_module_directories() -> Result, Error> { let mut module_dirs = HashMap::new(); - let cache_dir = std::path::Path::new(POWERSHELL_CACHE_DIR); + let cache_dir = std::path::Path::new(&*POWERSHELL_CACHE_DIR); if let Ok(entries) = fs::read_dir(cache_dir) { for entry in entries { @@ -391,7 +391,7 @@ pub async fn handle_powershell_job( .join(", "); let install_string = generate_powershell_install_code() - .replace("{path}", POWERSHELL_CACHE_DIR) + .replace("{path}", &*POWERSHELL_CACHE_DIR) .replace("{job_id}", &job.id.to_string()) .replace("{has_private_repo}", &format!("${has_private_repo}")) .replace("{has_credentials}", &format!("${has_credentials}")) @@ -442,7 +442,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); #[cfg(windows)] @@ -452,7 +452,7 @@ $PSModulePathBackup = $env:PSModulePath $env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{};$PSModulePathBackup\"", - POWERSHELL_CACHE_DIR + *POWERSHELL_CACHE_DIR ); // NOTE: powershell error handling / termination is quite tricky compared to bash @@ -525,7 +525,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR), )?; let cmd_args = vec![ "--config", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 047521a0b4..b289f05380 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -278,7 +278,7 @@ pub async fn uv_pip_compile( "requirements.txt", // Target to /tmp/windmill/cache/uv "--cache-dir", - UV_CACHE_DIR, + &*UV_CACHE_DIR, ]; args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]); @@ -805,7 +805,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) @@ -815,7 +815,7 @@ mount {{ "{ADDITIONAL_PYTHON_PATHS}", additional_python_paths_folders.as_str(), ) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), )?; } else { @@ -1410,10 +1410,10 @@ async fn spawn_uv_install( &nsjail_proto, NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT .replace("{WORKER_DIR}", worker_dir) - .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) + .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{TARGET_DIR}", &venv_p) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .as_str(), )?; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 52636808a6..78b03deb07 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -18,6 +18,8 @@ use windmill_common::{ use anyhow::{anyhow, bail}; use windmill_queue::append_logs; +#[cfg(unix)] +use crate::python_executor::UV_PATH; use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, @@ -25,8 +27,6 @@ use crate::{ HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, WIN_ENVS, }; -#[cfg(unix)] -use crate::python_executor::UV_PATH; impl From for PyVAlias { fn from(value: PyV) -> Self { @@ -234,7 +234,8 @@ impl PyV { pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; format!( - "{ROOT_CACHE_DIR}{}", + "{}{}", + *ROOT_CACHE_DIR, self.to_cache_dir_top_level(ignore_patch) ) } @@ -311,7 +312,7 @@ impl PyV { Command::new(uv_cmd) .env_clear() .envs(WIN_ENVS.to_vec()) - .env("UV_CACHE_DIR", UV_CACHE_DIR) + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) .args([ "python", "list", @@ -539,8 +540,8 @@ impl PyV { ]) // TODO: Do we need these? .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_CACHE_DIR", UV_CACHE_DIR), + ("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR), + ("UV_CACHE_DIR", &*UV_CACHE_DIR), ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -630,11 +631,9 @@ impl PyV { "--system", "--python-preference=only-managed", ]) - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_PYTHON_PREFERENCE", "only-managed"), - ("UV_CACHE_DIR", UV_CACHE_DIR), - ]) + .env("UV_PYTHON_INSTALL_DIR", &*PY_INSTALL_DIR) + .env("UV_PYTHON_PREFERENCE", "only-managed") + .env("UV_CACHE_DIR", &*UV_CACHE_DIR) // .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index 869b58d7de..11031fa481 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, process::Stdio}; use anyhow::anyhow; -use const_format::concatcp; use itertools::Itertools; use regex::Regex; use tokio::{ @@ -122,7 +121,7 @@ pub async fn prepare<'a>( .write_all(&wrap(inner_content)?.into_bytes()) .await?; - let mini_wm_path = format!("{RUBY_CACHE_DIR}/gems/windmill-internal/windmill"); + let mini_wm_path = format!("{}/gems/windmill-internal/windmill", *RUBY_CACHE_DIR); if !std::fs::metadata(&mini_wm_path).is_ok() { fs::create_dir_all(&mini_wm_path).await?; @@ -339,7 +338,7 @@ Your Gemfile syntax will continue to work as-is." &NSJAIL_CONFIG_LOCK_RUBY_CONTENT .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -588,7 +587,7 @@ async fn install<'a>( // 123...zx-activesupport-8.0.2 // ^^^^^^^^ hash based on source and type (GEM or GIT) let handle = format!("{}-{}-{}", hash, pkg, version); - let path = format!("{RUBY_CACHE_DIR}/gems/{}", &handle); + let path = format!("{}/gems/{}", *RUBY_CACHE_DIR, &handle); deps.push(RequiredDependency { path, @@ -632,7 +631,7 @@ async fn install<'a>( &NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -741,9 +740,9 @@ async fn install<'a>( }; // Include builtin windmill client { - const WM_INTERNAL: &str = concatcp!(RUBY_CACHE_DIR, "/gems/windmill-internal"); - res.top_level_paths.push(WM_INTERNAL.to_owned()); - res.rubylib += format!(":{WM_INTERNAL}").as_str(); + let wm_internal = format!("{}/gems/windmill-internal", *RUBY_CACHE_DIR); + res.top_level_paths.push(wm_internal.clone()); + res.rubylib += format!(":{wm_internal}").as_str(); } Ok(res) } @@ -800,7 +799,7 @@ mount {{ .replace("{JOB_DIR}", job_dir) .replace("{SHARED_MOUNT}", &shared_mount) .replace("{SHARED_DEPENDENCIES}", &shared_deps) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 54d90550b0..216c59f2cd 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -5,6 +5,7 @@ use std::{collections::HashMap, process::Stdio}; use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; +use crate::global_cache::save_cache; use itertools::Itertools; use tokio::{ fs::{create_dir_all, File}, @@ -16,7 +17,6 @@ use windmill_common::{ utils::calculate_hash, worker::{write_file, Connection}, }; -use crate::global_cache::save_cache; use windmill_queue::MiniPulledJob; use windmill_queue::{append_logs, CanceledBy}; @@ -337,7 +337,7 @@ async fn get_build_dir( if !is_sandboxing_enabled() { // If nsjail is disabled then entire worker has shared build directory // It drastically improves cache hit-rate. - Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true)) + Some((format!("{}/build/{worker_name}", *RUST_CACHE_DIR), true)) } else { // If nsjail is enabled, having global shared directory is vulnerability and target for an attack // Instead we either: @@ -345,7 +345,8 @@ async fn get_build_dir( // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all. Some(( format!( - "{RUST_CACHE_DIR}/build/{}@{}@{}", + "{}/build/{}@{}@{}", + *RUST_CACHE_DIR, &job.workspace_id, p.replace('/', "."), &job.created_by @@ -355,7 +356,10 @@ async fn get_build_dir( } } }) - .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false)); + .unwrap_or(( + format!("{}/build/{}", *RUST_CACHE_DIR, Uuid::new_v4()), + false, + )); { let (t, r, g) = ( @@ -449,7 +453,7 @@ pub async fn build_rust_crate( is_preview: bool, ) -> error::Result { ensure_rust_runtime_dirs(); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?; @@ -459,9 +463,9 @@ pub async fn build_rust_crate( "download.config.proto", &NSJAIL_CONFIG_COMPILE_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CARGO_HOME}", CARGO_HOME.as_str()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{BUILD}", &build_dir), )?; @@ -605,14 +609,13 @@ pub async fn handle_rust_job( check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; let hash = compute_rust_hash(inner_content, requirements_o); - let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); + let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let (cache, cache_logs) = - crate::global_cache::load_cache(&bin_path, &remote_path, false).await; + let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { let target = format!("{job_dir}/main"); @@ -669,10 +672,10 @@ pub async fn handle_rust_job( "run.config.proto", &NSJAIL_CONFIG_RUN_RUST_CONTENT .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CACHE_DIR}", &*RUST_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{TRACING_PROXY_CA_CERT_PATH}", TRACING_PROXY_CA_CERT_PATH) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), )?; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c354d34ee2..49a1048afd 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -37,7 +37,7 @@ use windmill_common::{ utils::{create_directory_async, WarnAfterExt}, worker::{ make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, - MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, + MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, WINDMILL_DIR, }, worker_group_job_stats::JobStatsMap, KillpillSender, @@ -47,7 +47,6 @@ use windmill_common::{ use windmill_common::ee_oss::LICENSE_KEY_VALID; use anyhow::Result; -use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; @@ -196,45 +195,47 @@ use windmill_common::bench::{benchmark_init, benchmark_verify, BenchmarkInfo, Be use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); +lazy_static::lazy_static! { + pub static ref PY310_CACHE_DIR: String = format!("{}python_3_10", *ROOT_CACHE_DIR); + pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR); + pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR); + pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR); -pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); + pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR); -pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); -pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); -pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); -pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); -pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); -pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); + pub static ref UV_CACHE_DIR: String = format!("{}uv", *ROOT_CACHE_DIR); + pub static ref PY_INSTALL_DIR: String = format!("{}py_runtime", *ROOT_CACHE_DIR); + pub static ref TAR_PYBASE_CACHE_DIR: String = format!("{}tar", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR: String = format!("{}deno", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_DEPS: String = format!("{}deno/deps", *ROOT_CACHE_DIR); + pub static ref DENO_CACHE_DIR_NPM: String = format!("{}deno/npm", *ROOT_CACHE_DIR); -pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); -pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); -pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu"); -pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); + pub static ref GO_CACHE_DIR: String = format!("{}go", *ROOT_CACHE_DIR); + pub static ref RUST_CACHE_DIR: String = format!("{}rust", *ROOT_CACHE_DIR); + pub static ref NU_CACHE_DIR: String = format!("{}nu", *ROOT_CACHE_DIR); + pub static ref CSHARP_CACHE_DIR: String = format!("{}csharp", *ROOT_CACHE_DIR); -// Java -pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java"); -pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache"); -pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository"); -pub const JAVA_HOME_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/home"); + // Java + pub static ref JAVA_CACHE_DIR: String = format!("{}java", *ROOT_CACHE_DIR); + pub static ref COURSIER_CACHE_DIR: String = format!("{}/coursier-cache", *JAVA_CACHE_DIR); + pub static ref JAVA_REPOSITORY_DIR: String = format!("{}/repository", *JAVA_CACHE_DIR); + pub static ref JAVA_HOME_DIR: String = format!("{}/home", *JAVA_CACHE_DIR); -// Ruby -pub const RUBY_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "ruby"); + // Ruby + pub static ref RUBY_CACHE_DIR: String = format!("{}ruby", *ROOT_CACHE_DIR); -// for related places search: ADD_NEW_LANG -pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); -pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); -pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle"); + // for related places search: ADD_NEW_LANG + pub static ref BUN_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_NOMOUNT_DIR); + pub static ref BUN_BUNDLE_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_DIR); + pub static ref BUN_CODEBASE_BUNDLE_CACHE_DIR: String = format!("{}script_bundle", *ROOT_CACHE_NOMOUNT_DIR); -pub const GO_BIN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "gobin"); -pub const POWERSHELL_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "powershell"); -pub const COMPOSER_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "composer"); + pub static ref GO_BIN_CACHE_DIR: String = format!("{}gobin", *ROOT_CACHE_DIR); + pub static ref POWERSHELL_CACHE_DIR: String = format!("{}powershell", *ROOT_CACHE_DIR); + pub static ref COMPOSER_CACHE_DIR: String = format!("{}composer", *ROOT_CACHE_DIR); -pub const TRACING_PROXY_CA_CERT_PATH: &str = - concatcp!(ROOT_CACHE_NOMOUNT_DIR, "tracing_proxy_ca.pem"); + pub static ref TRACING_PROXY_CA_CERT_PATH: String = + format!("{}tracing_proxy_ca.pem", *ROOT_CACHE_NOMOUNT_DIR); +} const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; @@ -1375,7 +1376,7 @@ pub async fn run_worker( let start_time = Instant::now(); - let worker_dir = format!("{TMP_DIR}/{worker_name}"); + let worker_dir = format!("{}/{worker_name}", *WINDMILL_DIR); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); #[cfg(feature = "python")]