diff --git a/backend/src/main.rs b/backend/src/main.rs index 1f1fab319d..a711a2653e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -65,8 +65,8 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, - GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, POWERSHELL_CACHE_DIR, RUST_CACHE_DIR, + TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -995,7 +995,6 @@ pub async fn run_workers(path: &str) -> Option { pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, + pub no_uv_install: bool, + pub no_uv_compile: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto new file mode 100644 index 0000000000..15e5c6c7f6 --- /dev/null +++ b/backend/windmill-worker/nsjail/download.py.pip.config.proto @@ -0,0 +1,93 @@ +name: "python download pip" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: 900 + +rlimit_as: 2048 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 64 + +envar: "HOME=/user" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: true +keep_env: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{WORKER_DIR}/download_deps.py.pip.sh" + dst: "/download_deps.sh" + is_bind: true +} + +mount { + src: "{CACHE_DIR}" + dst: "{CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +exec_bin { + path: "/bin/sh" + arg: "/download_deps.sh" +} + diff --git a/backend/windmill-worker/nsjail/download_deps.py.pip.sh b/backend/windmill-worker/nsjail/download_deps.py.pip.sh new file mode 100755 index 0000000000..efd6274832 --- /dev/null +++ b/backend/windmill-worker/nsjail/download_deps.py.pip.sh @@ -0,0 +1,24 @@ +#/bin/sh + +INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" ) +EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" ) +TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST") + +if [ ! -z "$INDEX_URL" ] +then + echo "\$INDEX_URL is set to $INDEX_URL" +fi + +if [ ! -z "$EXTRA_INDEX_URL" ] +then + echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL" +fi + +if [ ! -z "$TRUSTED_HOST" ] +then + echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" +fi + +CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +echo $CMD +eval $CMD diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index efd6274832..3898282d2a 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,6 +19,18 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" +CMD="/usr/local/bin/uv pip install +\"$REQ\" +--target \"$TARGET\" +--no-cache +--no-config +--no-color +--no-deps +--link-mode=copy +$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG +--index-strategy unsafe-best-match +--system +" + echo $CMD eval $CMD diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index abd9b68415..3548219ca5 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -116,6 +116,8 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), + true, + true, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 12584b905d..57b2b55885 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -20,13 +20,21 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, + no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; + use crate::PY311_CACHE_DIR; + tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); let folder_name = folder.split("/").last().unwrap(); - let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",); + let prefix = if no_uv { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; + let tar_path = format!("{prefix}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; let mut tar = tar::Builder::new(tar_file); @@ -46,7 +54,10 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!("/tar/pip/{folder_name}.tar")), + &Path::from(format!( + "/tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + )), std::fs::read(&tar_path)?.into(), ) .await @@ -71,7 +82,11 @@ pub async fn build_tar_and_push( } #[cfg(all(feature = "enterprise", feature = "parquet"))] -pub async fn pull_from_tar(client: Arc, folder: String) -> error::Result<()> { +pub async fn pull_from_tar( + client: Arc, + folder: String, + no_uv: bool, +) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; let folder_name = folder.split("/").last().unwrap(); @@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc, folder: String) -> erro tracing::info!("Attempting to pull piptar {folder_name} from bucket"); let start = Instant::now(); - let tar_path = format!("tar/pip/{folder_name}.tar"); + let tar_path = format!( + "tar/{}/{folder_name}.tar", + if no_uv { "pip" } else { "python_311" } + ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; // tracing::info!("B: {target} {folder}"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9db4ed529b..b06f30b98d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -42,6 +42,10 @@ lazy_static::lazy_static! { static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); + /// Use pip install + static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -50,6 +54,8 @@ lazy_static::lazy_static! { } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); +const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str = + include_str!("../nsjail/download.py.pip.config.proto"); const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); @@ -66,8 +72,8 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, - NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV, - UV_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, + PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, }; #[cfg(windows)] @@ -901,10 +907,10 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { - let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -929,8 +935,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotation.no_uv, - annotation.no_cache, + annotations.no_uv || annotations.no_uv_compile, + annotations.no_cache, ) .await .map_err(|e| { @@ -955,6 +961,8 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, + annotations.no_uv || annotations.no_uv_install, + false, ) .await?; additional_python_paths.append(&mut venv_path); @@ -966,6 +974,7 @@ lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); } +/// pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -977,12 +986,22 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // TODO: Remove (Deprecated) + mut no_uv_install: bool, + is_ansible: bool, ) -> error::Result> { let mut req_paths: Vec = vec![]; let mut vars = vec![("PATH", PATH_ENV.as_str())]; let pip_extra_index_url; let pip_index_url; + no_uv_install |= *USE_PIP_INSTALL; + + if no_uv_install && !is_ansible { + append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; + tracing::warn!("Fallback to pip"); + } + if !*DISABLE_NSJAIL { pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() @@ -1013,10 +1032,21 @@ pub async fn handle_python_reqs( let _ = write_file( job_dir, "download.config.proto", - &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - .replace("{WORKER_DIR}", &worker_dir) - .replace("{CACHE_DIR}", PIP_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + &(if no_uv_install { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK + } else { + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + }) + .replace("{WORKER_DIR}", &worker_dir) + .replace( + "{CACHE_DIR}", + if no_uv_install { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }, + ) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; }; @@ -1026,8 +1056,14 @@ pub async fn handle_python_reqs( if req.starts_with('#') { continue; } + let py_prefix = if no_uv_install { + PIP_CACHE_DIR + } else { + PY311_CACHE_DIR + }; + let venv_p = format!( - "{PIP_CACHE_DIR}/{}", + "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(&venv_p).await.is_ok() { @@ -1073,7 +1109,10 @@ pub async fn handle_python_reqs( .map(|(req, venv_p)| { let os = os.clone(); async move { - if pull_from_tar(os, venv_p.clone()).await.is_ok() { + if pull_from_tar(os, venv_p.clone(), no_uv_install) + .await + .is_ok() + { PullFromTar::Pulled(venv_p.to_string()) } else { PullFromTar::NotPulled(req.to_string(), venv_p.to_string()) @@ -1114,7 +1153,7 @@ pub async fn handle_python_reqs( for (req, venv_p) in req_with_penv { let mut logs1 = String::new(); - logs1.push_str("\n\n--- PIP INSTALL ---\n"); + logs1.push_str("\n\n--- UV PIP INSTALL ---\n"); logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses.")); append_logs(&job_id, w_id, logs1, db).await; @@ -1150,21 +1189,47 @@ pub async fn handle_python_reqs( #[cfg(windows)] let req = format!("{}", req); - let mut command_args = vec![ - PYTHON_PATH.as_str(), - "-m", - "pip", - "install", - &req, - "-I", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - venv_p.as_str(), - ]; + let mut command_args = if no_uv_install { + vec![ + PYTHON_PATH.as_str(), + "-m", + "pip", + "install", + &req, + "-I", + "--no-deps", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + venv_p.as_str(), + ] + } else { + vec![ + UV_PATH.as_str(), + "pip", + "install", + &req, + "--no-deps", + "--no-color", + // "-p", + // "3.11", + // Prevent uv from discovering configuration files. + "--no-config", + "--link-mode=copy", + // TODO: Doublecheck it + "--system", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + "--target", + venv_p.as_str(), + "--no-cache", + ] + }; let pip_extra_index_url = PIP_EXTRA_INDEX_URL .read() .await @@ -1196,7 +1261,7 @@ pub async fn handle_python_reqs( envs.push(("HOME", HOME_ENV.as_str())); - tracing::debug!("pip install command: {:?}", command_args); + tracing::debug!("uv pip install command: {:?}", command_args); #[cfg(unix)] { @@ -1207,7 +1272,12 @@ pub async fn handle_python_reqs( .envs(envs) .args([ "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + &format!( + "{}/{}-{}.lock", + LOCK_CACHE_DIR, + if no_uv_install { "pip" } else { "py311" }, + fssafe_req + ), "--command", &command_args.join(" "), ]) @@ -1218,16 +1288,20 @@ pub async fn handle_python_reqs( #[cfg(windows)] { - let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); - pip_cmd - .env_clear() + let installer_path = if no_uv_install { command_args[0] } else { "uv" }; + let mut cmd: Command = Command::new(&installer_path); + cmd.env_clear() .envs(envs) .envs(PROXY_ENVS.clone()) .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) .args(&command_args[1..]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + start_child_process(cmd, installer_path).await? } }; @@ -1240,7 +1314,7 @@ pub async fn handle_python_reqs( false, worker_name, &w_id, - &format!("pip install {req}"), + &format!("uv pip install {req}"), None, false, occupancy_metrics, @@ -1260,7 +1334,7 @@ pub async fn handle_python_reqs( tracing::warn!("S3 cache not available in the pro plan"); } else { let venv_p = venv_p.clone(); - tokio::spawn(build_tar_and_push(os, venv_p)); + tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install)); } } req_paths.push(venv_p); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 28a94d5f92..e6e92e1682 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs"); pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); +// Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); + +// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); +// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); + pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); @@ -257,6 +264,7 @@ const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh"); pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days @@ -311,6 +319,7 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() .and_then(|x| x.parse::().ok()) @@ -740,6 +749,13 @@ pub async fn run_worker