diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index c54d3c05a7..7838f648ec 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -6,12 +6,13 @@ use serde_json::{json, Value}; use tokio::process::Command; use windmill_common::{error::Error, jobs::QueuedJob}; +const BIN_BASH: &str = "/bin/bash"; const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); use crate::{ common::{ get_reserved_variables, handle_child, read_file, read_file_content, set_logs, - transform_json_value, write_file, + start_child_process, transform_json_value, write_file, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -89,7 +90,8 @@ pub async fn handle_bash_job( .await?; let mut cmd_args = vec!["--config", "run.config.proto", "--", "/bin/bash", "main.sh"]; cmd_args.extend(args); - Command::new(NSJAIL_PATH.as_str()) + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd .current_dir(job_dir) .env_clear() .envs(reserved_variables) @@ -97,12 +99,13 @@ pub async fn handle_bash_job( .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let mut cmd_args = vec!["main.sh"]; cmd_args.extend(&args); - Command::new("/bin/bash") + let mut bash_cmd = Command::new(BIN_BASH); + bash_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -112,8 +115,8 @@ pub async fn handle_bash_job( .env("HOME", HOME_ENV.as_str()) .args(cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(bash_cmd, BIN_BASH).await? }; handle_child( &job.id, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index a02f1ea8c5..79aa96e059 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -17,7 +17,7 @@ use crate::{common::build_envs_map, JobCompleted}; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs, - write_file, write_file_binary, + start_child_process, write_file, write_file_binary, }, AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -95,20 +95,21 @@ pub async fn gen_lockfile( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(&base_internal_url).await; - let child = Command::new(&*BUN_PATH) + let mut child_cmd = Command::new(&*BUN_PATH); + child_cmd .current_dir(job_dir) .env_clear() .envs(common_bun_proc_envs.clone()) .args(vec!["run", "build.ts"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, &*BUN_PATH).await?; handle_child( job_id, db, logs, - child, + child_process, false, worker_name, w_id, @@ -193,20 +194,21 @@ pub async fn install_lockfile( worker_name: &str, common_bun_proc_envs: HashMap, ) -> Result<()> { - let child = Command::new(&*BUN_PATH) + let mut child_cmd = Command::new(&*BUN_PATH); + child_cmd .current_dir(job_dir) .env_clear() .envs(common_bun_proc_envs) .args(vec!["install"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, &*BUN_PATH).await?; handle_child( job_id, db, logs, - child, + child_process, false, worker_name, w_id, @@ -439,7 +441,8 @@ plugin(p) ) .await?; - Command::new(NSJAIL_PATH.as_str()) + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -459,8 +462,8 @@ plugin(p) "/tmp/bun/wrapper.ts", ]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let script_path = format!("{job_dir}/wrapper.ts"); let args = vec![ @@ -471,7 +474,8 @@ plugin(p) "./loader.bun.ts", &script_path, ]; - Command::new(&*BUN_PATH) + let mut bun_cmd = Command::new(&*BUN_PATH); + bun_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -479,8 +483,8 @@ plugin(p) .envs(common_bun_proc_envs) .args(args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(bun_cmd, &*BUN_PATH).await? }; handle_child( @@ -720,7 +724,8 @@ plugin(p) "./loader.bun.ts", &script_path, ]; - Command::new(&*BUN_PATH) + let mut bun_cmd = Command::new(&*BUN_PATH); + bun_cmd .current_dir(job_dir) .env_clear() .envs(context_envs) @@ -735,8 +740,8 @@ plugin(p) .args(args) .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(bun_cmd, &*BUN_PATH).await? }; let stdout = child diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 8b1af6442a..d3ce1fe3ac 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -3,6 +3,7 @@ use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use serde_json::{json, Value}; use sqlx::{Pool, Postgres}; +use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; use windmill_api_client::{types::CreateResource, Client}; use windmill_common::worker::CLOUD_HOSTED; @@ -595,6 +596,12 @@ pub async fn handle_child( } } +pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result { + return cmd + .spawn() + .map_err(|err| tentatively_improve_error(Error::IoErr(err), executable)); +} + /// takes stdout and stderr from Child, panics if either are not present /// /// builds a stream joining both stdout and stderr each read line by line @@ -695,6 +702,16 @@ async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef, db: impl Borrow< } } +fn tentatively_improve_error(err: Error, executable: &str) -> Error { + if err + .to_string() + .contains("No such file or directory (os error 2)") + { + return Error::InternalErr(format!("Executable {executable} not found on worker")); + } + return err; +} + pub async fn clean_cache() -> error::Result<()> { tracing::info!("Started cleaning cache"); tokio::fs::remove_dir_all(ROOT_CACHE_DIR).await?; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 106e498a79..9e041ad9b2 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs, - write_file, + start_child_process, write_file, }, AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, @@ -96,7 +96,8 @@ pub async fn generate_deno_lock( if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() { deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone()); } - let child = Command::new(DENO_PATH.as_str()) + let mut child_cmd = Command::new(DENO_PATH.as_str()); + child_cmd .current_dir(job_dir) .args(vec![ "cache", @@ -109,14 +110,14 @@ pub async fn generate_deno_lock( ]) .envs(deno_envs) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, DENO_PATH.as_str()).await?; handle_child( job_id, db, logs, - child, + child_process, false, worker_name, w_id, @@ -302,7 +303,8 @@ run().catch(async (e) => {{ args.push("-A"); } args.push(&script_path); - Command::new(DENO_PATH.as_str()) + let mut deno_cmd = Command::new(DENO_PATH.as_str()); + deno_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -311,8 +313,8 @@ run().catch(async (e) => {{ .env("DENO_DIR", DENO_CACHE_DIR) .args(args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(deno_cmd, DENO_PATH.as_str()).await? }; // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); // start = Instant::now(); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index d1fbf34439..11ff53892e 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -17,7 +17,7 @@ use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE}; use crate::{ common::{ capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result, - set_logs, write_file, + set_logs, start_child_process, write_file, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, @@ -177,7 +177,8 @@ func Run(req Req) (interface{{}}, error){{ } } - let build_go = Command::new(GO_PATH.as_str()) + let mut build_go_cmd = Command::new(GO_PATH.as_str()); + build_go_cmd .current_dir(job_dir) .env_clear() .env("PATH", PATH_ENV.as_str()) @@ -186,13 +187,13 @@ func Run(req Req) (interface{{}}, error){{ .env("HOME", HOME_ENV.as_str()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?; handle_child( &job.id, db, logs, - build_go, + build_go_process, false, worker_name, &job.workspace_id, @@ -228,7 +229,8 @@ func Run(req Req) (interface{{}}, error){{ .replace("{SHARED_MOUNT}", shared_mount), ) .await?; - Command::new(NSJAIL_PATH.as_str()) + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -238,10 +240,11 @@ func Run(req Req) (interface{{}}, error){{ .env("BASE_INTERNAL_URL", base_internal_url) .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let mut run_go = Command::new("./main"); + let compiled_executable_name = "./main"; + let mut run_go = Command::new(compiled_executable_name); run_go .current_dir(job_dir) .env_clear() @@ -260,10 +263,8 @@ func Run(req Req) (interface{{}}, error){{ run_go.env("GOPROXY", goproxy); } - run_go - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + run_go.stdout(Stdio::piped()).stderr(Stdio::piped()); + start_child_process(run_go, compiled_executable_name).await? }; handle_child( &job.id, @@ -316,18 +317,19 @@ pub async fn install_go_dependencies( ) -> error::Result { if !skip_go_mod { gen_go_mymod(code, job_dir).await?; - let child = Command::new("go") + let mut child_cmd = Command::new(GO_PATH.as_str()); + child_cmd .current_dir(job_dir) .args(vec!["mod", "init", "mymod"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, GO_PATH.as_str()).await?; handle_child( job_id, db, logs, - child, + child_process, false, worker_name, w_id, @@ -377,18 +379,20 @@ pub async fn install_go_dependencies( } let mod_command = if skip_tidy { "download" } else { "tidy" }; - let child = Command::new(GO_PATH.as_str()) + let mut child_cmd = Command::new(GO_PATH.as_str()); + child_cmd .current_dir(job_dir) .env("GOPATH", GO_CACHE_DIR) .args(vec!["mod", mod_command]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, GO_PATH.as_str()).await?; + handle_child( job_id, db, logs, - child, + child_process, false, worker_name, &w_id, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 97ba432c12..50efd3e990 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -46,7 +46,7 @@ use crate::S3_CACHE_BUCKET; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs, - write_file, + start_child_process, write_file, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, TZ_ENV, @@ -117,17 +117,18 @@ pub async fn pip_compile( args.extend(["--trusted-host", host]); } - let child = Command::new("pip-compile") + let mut child_cmd = Command::new("pip-compile"); + child_cmd .current_dir(job_dir) .args(args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "pip-compile").await?; handle_child( job_id, db, logs, - child, + child_process, false, worker_name, &w_id, @@ -429,7 +430,8 @@ mount {{ job.id ); let child = if !*DISABLE_NSJAIL { - Command::new(NSJAIL_PATH.as_str()) + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd .current_dir(job_dir) .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf @@ -447,10 +449,11 @@ mount {{ "wrapper", ]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - Command::new(PYTHON_PATH.as_str()) + let mut python_cmd = Command::new(PYTHON_PATH.as_str()); + python_cmd .current_dir(job_dir) .env_clear() .envs(envs) @@ -460,8 +463,8 @@ mount {{ .env("BASE_INTERNAL_URL", base_internal_url) .args(vec!["-u", "-m", "wrapper"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; handle_child( @@ -569,14 +572,15 @@ pub async fn handle_python_reqs( let req = req.to_string(); vars.push(("REQ", &req)); vars.push(("TARGET", &venv_p)); - Command::new(NSJAIL_PATH.as_str()) + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd .current_dir(job_dir) .env_clear() .envs(vars) .args(vec!["--config", "download.config.proto"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let mut command_args = vec![ PYTHON_PATH.as_str(), @@ -614,7 +618,8 @@ pub async fn handle_python_reqs( envs.push(("NO_PROXY", no_proxy)); } - Command::new(FLOCK_PATH.as_str()) + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd .env_clear() .envs(envs) .args([ @@ -624,8 +629,8 @@ pub async fn handle_python_reqs( &command_args.join(" "), ]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? }; let child = handle_child(