diff --git a/backend/windmill-worker/nsjail/download.rust.config.proto b/backend/windmill-worker/nsjail/download.rust.config.proto new file mode 100644 index 0000000000..7239a77b74 --- /dev/null +++ b/backend/windmill-worker/nsjail/download.rust.config.proto @@ -0,0 +1,117 @@ +name: "rust download script" + +mode: ONCE +hostname: "rust" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false + +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} +mount { + src: "{JOB_DIR}" + dst: "/tmp" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +mount { + src: "{BUILD}" + dst: "{BUILD}" + is_bind: true + mandatory: false + rw: true +} +mount { + src: "{CARGO_HOME}" + dst: "{CARGO_HOME}" + is_bind: true + # Readonly + rw: false +} + +mount { + src: "{BUILD}/registry" + dst: "{CARGO_HOME}/registry" + is_bind: true + mandatory: true + # Read-write + rw: true +} + +mount { + src: "{BUILD}/git" + dst: "{CARGO_HOME}/git" + is_bind: true + mandatory: true + # Read-write + rw: true +} + +{DEV} diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 3357cd88a9..d5265f8ddd 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -104,3 +104,6 @@ mount { } {SHARED_MOUNT} + +{DEV} + diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 1a6a93ecca..0255783f6f 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -4,7 +4,11 @@ use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; use itertools::Itertools; -use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use tokio::{ + fs::{create_dir_all, File}, + io::AsyncReadExt, + process::Command, +}; use windmill_common::{ error::{self, Error}, utils::calculate_hash, @@ -19,8 +23,8 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - RUST_CACHE_DIR, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, + TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -28,12 +32,18 @@ use windmill_common::client::AuthedClient; use crate::SYSTEM_ROOT; const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); +const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = + include_str!("../nsjail/download.rust.config.proto"); lazy_static::lazy_static! { static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = std::env::var("CARGO_PATH").unwrap_or_else(|_| format!("{}/bin/cargo", CARGO_HOME.as_str())); + // static ref CARGO_SWEEP_PATH: String = std::env::var("CARGO_SWEEP_PATH").unwrap_or_else(|_| format!("{}/bin/cargo-sweep", CARGO_HOME.as_str())); + static ref SWEEP_MAXSIZE: String = std::env::var("CARGO_SWEEP_MAXSIZE").unwrap_or("25GB".to_owned()); + static ref NO_SHARED_BUILD_DIR: bool = std::env::var("RUST_NO_SHARED_BUILD_DIR").ok().map(|flag| flag == "true").unwrap_or(false); + } #[cfg(windows)] @@ -42,7 +52,20 @@ lazy_static::lazy_static! { static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); } -#[cfg(unix)] +#[cfg(debug_assertions)] +const DEV_CONF_NSJAIL: &'static str = r#" +# Mount nix store for nixos to work properly +mount { + src: "/nix/store" + dst: "/nix/store" + is_bind: true + mandatory: false +} +"#; + +#[cfg(not(debug_assertions))] +const DEV_CONF_NSJAIL: &'static str = ""; + lazy_static::lazy_static! { static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); @@ -176,54 +199,211 @@ pub async fn generate_cargo_lockfile( Ok(req_content) } +async fn get_build_dir( + job: &MiniPulledJob, + job_dir: &str, + conn: &Connection, + worker_name: &str, + is_preview: bool, +) -> anyhow::Result { + let (bd, run_sweep) = job + .runnable_path + .as_ref() + .and_then(|p| { + if !is_preview || *NO_SHARED_BUILD_DIR { + None + } else { + if *DISABLE_NSJAIL { + // If nsjail is disabled then entire worker has shared build directory + // It drastically improves cache hit-rate. + Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true)) + } else { + // If nsjail is enabled, having global shared directory is vulnerability and target for an attack + // Instead we either: + // 1. Create different build directory for workspace script and user. Balanced caching while mainining high degree of security. + // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all. + Some(( + format!( + "{RUST_CACHE_DIR}/build/{}@{}@{}", + &job.workspace_id, + p.replace('/', "."), + &job.created_by + ), + true, + )) + } + } + }) + .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false)); + + { + let (t, r, g) = ( + create_dir_all(format!("{}/target", &bd)).await, + create_dir_all(format!("{}/registry", &bd)).await, + create_dir_all(format!("{}/git", &bd)).await, + ); + + t.and(r) + .and(g) + .map_err(|e| anyhow::anyhow!("Could not create build dir for rust.\ne: {e}"))?; + } + + if run_sweep { + // Also run sweep to make sure target isn't using too much disk + let mut sweep_cmd = Command::new(CARGO_PATH.as_str()); + sweep_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_TARGET_DIR", &(bd.clone() + "/target")) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .args(["sweep", "--maxsize", SWEEP_MAXSIZE.as_str()]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + sweep_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + sweep_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + sweep_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } + + let (job_id, conn, w_id, wk_name) = ( + job.id.clone(), + conn.clone(), + job.workspace_id.clone(), + worker_name.to_owned(), + ); + + tokio::spawn(async move { + if let Err(e) = match start_child_process(sweep_cmd, CARGO_PATH.as_str()).await { + Ok(sweep_process) => { + handle_child( + &job_id, + &conn, + &mut 0, + &mut None, + sweep_process, + false, + &wk_name, + &w_id, + "cargo sweep", + None, + false, + &mut None, + None, + ) + .await + } + Err(e) => Err(e), + } { + tracing::warn!( + workspace_id = %w_id, + job_id = %job_id, + "Failed to run `cargo sweep`. Rust cache may grow over time, cargo sweep is meant to clean up unused cache.\ne: {e}\n" + ); + } + }); + } + Ok(bd) +} + pub async fn build_rust_crate( - job_id: &Uuid, + job: &MiniPulledJob, mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, conn: &Connection, worker_name: &str, - w_id: &str, base_internal_url: &str, hash: &str, occupancy_metrics: &mut OccupancyMetrics, + is_preview: bool, ) -> error::Result { let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); - let mut build_rust_cmd = Command::new(CARGO_PATH.as_str()); - build_rust_cmd - .current_dir(job_dir) - .env_clear() - .envs(PROXY_ENVS.clone()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("CARGO_HOME", CARGO_HOME.as_str()) - .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) - .args(vec!["build", "--release"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?; - #[cfg(windows)] - { - build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); - build_rust_cmd.env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), - ); - build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); - } + let child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "download.config.proto", + &NSJAIL_CONFIG_COMPILE_RUST_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CARGO_HOME}", CARGO_HOME.as_str()) + .replace("{DEV}", DEV_CONF_NSJAIL) + .replace("{BUILD}", &build_dir), + )?; + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(PROXY_ENVS.clone()) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target")) + .args(vec![ + "--config", + "download.config.proto", + "--", + CARGO_PATH.as_ref(), + "build", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if !is_preview { + nsjail_cmd.arg("--release"); + } + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { + let mut build_rust_cmd = Command::new(CARGO_PATH.as_str()); + build_rust_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target")) + .args(vec!["build"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); - let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; + if !is_preview { + build_rust_cmd.arg("--release"); + } + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } + start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await? + }; handle_child( - job_id, + &job.id, conn, mem_peak, canceled_by, - build_rust_process, + child, false, worker_name, - w_id, + &job.workspace_id, "rust build", None, false, @@ -231,16 +411,19 @@ pub async fn build_rust_crate( None, ) .await?; - append_logs(job_id, w_id, "\n\n", conn).await; + append_logs(&job.id, &job.workspace_id, "\n\n", conn).await; tokio::fs::copy( - &format!("{job_dir}/target/release/main"), + &format!( + "{build_dir}/target/{}/main", + if is_preview { "debug" } else { "release" }, + ), format! {"{job_dir}/main"}, ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/release/main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" )) })?; @@ -298,6 +481,9 @@ pub async fn handle_rust_job( let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; @@ -332,16 +518,16 @@ pub async fn handle_rust_job( create_args_and_out_file(client, job, job_dir, conn).await?; build_rust_crate( - &job.id, + &job, mem_peak, canceled_by, job_dir, conn, worker_name, - &job.workspace_id, base_internal_url, &hash, occupancy_metrics, + requirements_o.is_none(), ) .await? }; @@ -349,9 +535,6 @@ pub async fn handle_rust_job( let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n"); append_logs(&job.id, &job.workspace_id, logs2, conn).await; - let reserved_variables = - get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let child = if !*DISABLE_NSJAIL { let _ = write_file( job_dir, @@ -361,6 +544,7 @@ pub async fn handle_rust_job( .replace("{CACHE_DIR}", RUST_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 78eec753d1..ddc1cdb3af 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -3,6 +3,7 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 138038559c..7ed7108de2 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -22,6 +22,7 @@ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index e51e73a3ea..cd17557256 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev # Rust COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/flake.nix b/flake.nix index bb193a318f..10a1e26c14 100644 --- a/flake.nix +++ b/flake.nix @@ -81,6 +81,7 @@ buildInputs = buildInputs ++ (with pkgs; [ # Essentials rust + cargo-sweep git xcaddy sqlx-cli @@ -187,6 +188,7 @@ # for related places search: ADD_NEW_LANG FLOCK_PATH = "${pkgs.flock}/bin/flock"; CARGO_PATH = "${rust}/bin/cargo"; + CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep"; DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet"; DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet"; ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib";