From 50bc4caf9f0ffa0f833e261edbf9d11f7253e46d Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Wed, 19 Feb 2025 18:33:28 +0300 Subject: [PATCH] feat: remove `pip` fallback option for python and ansible (#5186) * refactor!: Remove `pip` fallback option for python and ansible BREAKING CHANGE: pip was deprecated since 1.425.0 (2024-11-15) * fix errors in main.rs * fix tests * remove nsjail for pip * fix imports * fix compilation error --- backend/src/main.rs | 9 +- backend/tests/worker.rs | 13 +- backend/windmill-common/src/worker.rs | 3 - .../nsjail/download.py.pip.config.proto | 86 ----- .../nsjail/download_deps.py.pip.sh | 24 -- .../windmill-worker/src/ansible_executor.rs | 2 - backend/windmill-worker/src/global_cache.rs | 20 +- .../windmill-worker/src/python_executor.rs | 314 ++++-------------- backend/windmill-worker/src/worker.rs | 13 - .../windmill-worker/src/worker_lockfiles.rs | 41 +-- 10 files changed, 77 insertions(+), 448 deletions(-) delete mode 100644 backend/windmill-worker/nsjail/download.py.pip.config.proto delete mode 100755 backend/windmill-worker/nsjail/download_deps.py.pip.sh diff --git a/backend/src/main.rs b/backend/src/main.rs index ab74895ca8..afd7ea08f4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -65,9 +65,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, - LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, + RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, + TAR_PY313_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1041,10 +1041,8 @@ pub async fn run_workers( let mut handles = Vec::with_capacity(num_workers as usize); for x in [ - LOCK_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, - TAR_PIP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, @@ -1057,7 +1055,6 @@ pub async fn run_workers( TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, - PIP_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, GO_CACHE_DIR, GO_BIN_CACHE_DIR, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index e1f73f5871..31fbeef7ea 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1015,15 +1015,10 @@ fn spawn_test_worker( tokio::sync::broadcast::Sender<()>, tokio::task::JoinHandle<()>, ) { - for x in [ - windmill_worker::LOCK_CACHE_DIR, - windmill_worker::GO_BIN_CACHE_DIR, - ] { - std::fs::DirBuilder::new() - .recursive(true) - .create(x) - .expect("could not create initial worker dir"); - } + std::fs::DirBuilder::new() + .recursive(true) + .create(windmill_worker::GO_BIN_CACHE_DIR) + .expect("could not create initial worker dir"); let (tx, rx) = tokio::sync::broadcast::channel(1); let db = db.to_owned(); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index eb5bd1d6f1..f07226265a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -362,9 +362,6 @@ fn parse_file(path: &str) -> Option { #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, - pub no_uv: bool, - pub no_uv_install: bool, - pub no_uv_compile: bool, pub no_postinstall: bool, pub py310: bool, pub py311: bool, diff --git a/backend/windmill-worker/nsjail/download.py.pip.config.proto b/backend/windmill-worker/nsjail/download.py.pip.config.proto deleted file mode 100644 index 6e2a8a1974..0000000000 --- a/backend/windmill-worker/nsjail/download.py.pip.config.proto +++ /dev/null @@ -1,86 +0,0 @@ -name: "python download pip" - -mode: ONCE -hostname: "python" -log_level: ERROR -time_limit: 900 - -rlimit_as: 2048 -rlimit_cpu: 1000 -rlimit_fsize: 1024 -rlimit_nofile: 64 - -envar: "HOME=/user" -envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" - -cwd: "/tmp" - -clone_newnet: false -clone_newuser: {CLONE_NEWUSER} - -keep_caps: true -keep_env: true - -mount { - src: "/bin" - dst: "/bin" - is_bind: true -} - -mount { - src: "/lib" - dst: "/lib" - is_bind: true -} - -mount { - src: "/lib64" - dst: "/lib64" - is_bind: true - mandatory: false -} - -mount { - src: "/usr" - dst: "/usr" - is_bind: true -} - -mount { - src: "/etc" - dst: "/etc" - is_bind: true -} - -mount { - src: "/dev/null" - dst: "/dev/null" - is_bind: true - rw: true -} - -mount { - dst: "/tmp" - fstype: "tmpfs" - rw: true - options: "size=500000000" -} - - -mount { - src: "{WORKER_DIR}/download_deps.py.pip.sh" - dst: "/download_deps.sh" - is_bind: true -} - -mount { - src: "/dev/urandom" - dst: "/dev/urandom" - is_bind: true -} - -exec_bin { - path: "/bin/sh" - arg: "/download_deps.sh" -} - diff --git a/backend/windmill-worker/nsjail/download_deps.py.pip.sh b/backend/windmill-worker/nsjail/download_deps.py.pip.sh deleted file mode 100755 index efd6274832..0000000000 --- a/backend/windmill-worker/nsjail/download_deps.py.pip.sh +++ /dev/null @@ -1,24 +0,0 @@ -#/bin/sh - -INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" ) -EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" ) -TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST") - -if [ ! -z "$INDEX_URL" ] -then - echo "\$INDEX_URL is set to $INDEX_URL" -fi - -if [ ! -z "$EXTRA_INDEX_URL" ] -then - echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL" -fi - -if [ ! -z "$TRUSTED_HOST" ] -then - echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" -fi - -CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG" -echo $CMD -eval $CMD diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index fee2eaa119..d3fee3a781 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -86,7 +86,6 @@ async fn handle_ansible_python_deps( &mut Some(occupancy_metrics), PyVersion::Py311, false, - false, ) .await .map_err(|e| { @@ -113,7 +112,6 @@ async fn handle_ansible_python_deps( worker_dir, &mut Some(occupancy_metrics), crate::python_executor::PyVersion::Py311, - false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 8239ab77d4..3bc652d8be 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -22,11 +22,10 @@ pub async fn build_tar_and_push( folder: String, // python_311 python_xyz: String, - no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; - use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR}; + use crate::TAR_PYBASE_CACHE_DIR; tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); @@ -34,11 +33,7 @@ pub async fn build_tar_and_push( // e.g. tiny==1.0.0 let folder_name = folder.split("/").last().unwrap(); - let prefix = if no_uv { - TAR_PIP_CACHE_DIR - } else { - &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz) - }; + let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz); let tar_path = format!("{prefix}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; @@ -59,10 +54,7 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!( - "/tar/{TARGET}/{}/{folder_name}.tar", - if no_uv { "pip" } else { &python_xyz } - )), + &Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")), std::fs::read(&tar_path)?.into(), ) .await @@ -92,7 +84,6 @@ pub async fn pull_from_tar( folder: String, // python_311 python_xyz: String, - no_uv: bool, ) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; @@ -102,10 +93,7 @@ pub async fn pull_from_tar( let start = Instant::now(); - let tar_path = format!( - "tar/{TARGET}/{}/{folder_name}.tar", - if no_uv { "pip".to_owned() } else { python_xyz } - ); + let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar"); let bytes = attempt_fetch_bytes(client, &tar_path).await?; extract_tar(bytes, &folder).await.map_err(|e| { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 531c41887a..d063e1aca7 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -48,31 +48,18 @@ lazy_static::lazy_static! { static ref PY_CONCURRENT_DOWNLOADS: usize = var("PY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); - static ref FLOCK_PATH: String = - var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string()); static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap(); static ref TRUSTED_HOST: Option = var("PY_TRUSTED_HOST").ok().or(var("PIP_TRUSTED_HOST").ok()); static ref INDEX_CERT: Option = var("PY_INDEX_CERT").ok().or(var("PIP_INDEX_CERT").ok()); static ref NATIVE_CERT: bool = var("PY_NATIVE_CERT").ok().or(var("UV_NATIVE_TLS").ok()).map(|flag| flag == "true").unwrap_or(false); - pub static ref USE_SYSTEM_PYTHON: bool = var("USE_SYSTEM_PYTHON") - .ok().map(|flag| flag == "true").unwrap_or(false); - - pub static ref USE_PIP_COMPILE: bool = var("USE_PIP_COMPILE") - .ok().map(|flag| flag == "true").unwrap_or(false); - - pub static ref USE_PIP_INSTALL: bool = var("USE_PIP_INSTALL") - .ok().map(|flag| flag == "true").unwrap_or(false); - static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); static ref EPHEMERAL_TOKEN_CMD: Option = var("EPHEMERAL_TOKEN_CMD").ok(); } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); -const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str = - include_str!("../nsjail/download.py.pip.config.proto"); const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); @@ -89,8 +76,8 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, - LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + UV_CACHE_DIR, }; // To change latest stable version: @@ -451,8 +438,6 @@ pub async fn uv_pip_compile( py_version: PyVersion, // Debug-only flag no_cache: bool, - // Fallback to pip-compile. Will be removed in future - mut no_uv: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); @@ -495,19 +480,8 @@ pub async fn uv_pip_compile( #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; - let mut req_hash = format!("py-{}", calculate_hash(&requirements)); + let req_hash = format!("py-{}", calculate_hash(&requirements)); - if no_uv || *USE_PIP_COMPILE { - logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); - // Set no_uv if not setted - no_uv = true; - // Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable. - // Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db). - // It will recalculate lockfile even if inputs have not been changed. - req_hash.push_str("-no_uv"); - // Will be in format: - // py-000..000-no_uv - } if !no_cache { if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", @@ -530,76 +504,7 @@ pub async fn uv_pip_compile( write_file(job_dir, file, &requirements)?; - // Fallback pip-compile. Will be removed in future - if no_uv { - tracing::debug!("Fallback to pip-compile"); - - let mut args = vec![ - "-q", - "--no-header", - file, - "--resolver=backtracking", - "--strip-extras", - ]; - let mut pip_args = vec![]; - let pip_extra_index_url = PIP_EXTRA_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_extra_index_url.as_ref() { - url.split(",").for_each(|url| { - args.extend(["--extra-index-url", url]); - pip_args.push(format!("--extra-index-url {}", url)); - }); - args.push("--no-emit-index-url"); - } - let pip_index_url = PIP_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_index_url.as_ref() { - args.extend(["--index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--index-url {}", url)); - } - if let Some(host) = TRUSTED_HOST.as_ref() { - args.extend(["--trusted-host", host]); - } - if let Some(cert_path) = INDEX_CERT.as_ref() { - args.extend(["--cert", cert_path]); - } - let pip_args_str = pip_args.join(" "); - if pip_args.len() > 0 { - args.extend(["--pip-args", &pip_args_str]); - } - tracing::debug!("pip-compile args: {:?}", args); - - let mut child_cmd = Command::new("pip-compile"); - child_cmd - .current_dir(job_dir) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "pip-compile").await?; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - canceled_by, - child_process, - false, - worker_name, - &w_id, - "pip-compile", - None, - false, - occupancy_metrics, - ) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; - } else { + { // Make sure we have python runtime installed py_version .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) @@ -904,11 +809,9 @@ pub async fn handle_python_job( ) .await?; - let PythonAnnotations { no_uv, no_postinstall, .. } = PythonAnnotations::parse(inner_content); + let PythonAnnotations { no_postinstall, .. } = PythonAnnotations::parse(inner_content); tracing::debug!("Finished handling python dependencies"); - let python_path = if no_uv { - PYTHON_PATH.clone() - } else if let Some(python_path) = py_version + let python_path = if let Some(python_path) = py_version .get_python( &job.id, mem_peak, @@ -931,15 +834,7 @@ pub async fn handle_python_job( tracing::debug!("Finished deps postinstall stage"); } - if no_uv { - append_logs( - &job.id, - &job.workspace_id, - format!("\n\n--- SYSTEM PYTHON (Fallback) CODE EXECUTION ---\n",), - db, - ) - .await; - } else { + { append_logs( &job.id, &job.workspace_id, @@ -1525,7 +1420,6 @@ async fn handle_python_deps( occupancy_metrics, annotated_pyv.unwrap_or(instance_pyv), annotations.no_cache, - annotations.no_uv || annotations.no_uv_compile, ) .await .map_err(|e| { @@ -1587,7 +1481,6 @@ async fn handle_python_deps( worker_dir, occupancy_metrics, final_version, - annotations.no_uv || annotations.no_uv_install, ) .await?; additional_python_paths.append(&mut venv_path); @@ -1611,7 +1504,6 @@ async fn spawn_uv_install( (pip_extra_index_url, pip_index_url): (Option, Option), // If none, it is system python py_path: Option, - no_uv_install: bool, worker_dir: &str, ) -> Result { if !*DISABLE_NSJAIL { @@ -1653,15 +1545,12 @@ async fn spawn_uv_install( let _ = write_file( job_dir, &nsjail_proto, - &(if no_uv_install { - NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK - } else { - NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT - }) - .replace("{WORKER_DIR}", worker_dir) - .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) - .replace("{TARGET_DIR}", &venv_p) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + .replace("{WORKER_DIR}", worker_dir) + .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) + .replace("{TARGET_DIR}", &venv_p) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .as_str(), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -1675,72 +1564,47 @@ async fn spawn_uv_install( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await } else { - let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); #[cfg(unix)] - let req = if no_uv_install { - format!("'{}'", req) - } else { - req.to_owned() - }; + let req = req.to_owned(); #[cfg(windows)] let req = format!("{}", req); - let mut command_args = if no_uv_install { - vec![ - PYTHON_PATH.as_str(), - "-m", - "pip", - "install", - &req, - "-I", - "--no-deps", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - venv_p, - ] - } else { - vec![ - UV_PATH.as_str(), - "pip", - "install", - &req, - "--no-deps", - "--no-color", - // Prevent uv from discovering configuration files. - "--no-config", - "--link-mode=copy", - "--system", - // Prefer main index over extra - // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes - // TODO: Use env variable that can be toggled from UI - "--index-strategy", - "unsafe-best-match", - "--target", - venv_p, - "--no-cache", - // If we invoke uv pip install, then we want to overwrite existing data - "--reinstall", - ] - }; + let mut command_args = vec![ + UV_PATH.as_str(), + "pip", + "install", + &req, + "--no-deps", + "--no-color", + // Prevent uv from discovering configuration files. + "--no-config", + "--link-mode=copy", + "--system", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + "--target", + venv_p, + "--no-cache", + // If we invoke uv pip install, then we want to overwrite existing data + "--reinstall", + ]; - if !no_uv_install { - if let Some(py_path) = py_path.as_ref() { - command_args.extend([ - "-p", - py_path.as_str(), - "--python-preference", - "only-managed", // - ]); - } else { - command_args.extend([ - "--python-preference", - "only-system", // - ]); - } + if let Some(py_path) = py_path.as_ref() { + command_args.extend([ + "-p", + py_path.as_str(), + "--python-preference", + "only-managed", // + ]); + } else { + command_args.extend([ + "--python-preference", + "only-system", // + ]); } if let Some(url) = pip_extra_index_url.as_ref() { @@ -1774,36 +1638,14 @@ async fn spawn_uv_install( #[cfg(unix)] { - if no_uv_install { - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(PROXY_ENVS.clone()) - .envs(envs) - .args([ - "-x", - &format!( - "{}/{}-{}.lock", - LOCK_CACHE_DIR, - if no_uv_install { "pip" } else { "py311" }, - fssafe_req - ), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await - } else { - let mut cmd = Command::new(command_args[0]); - cmd.env_clear() - .envs(PROXY_ENVS.clone()) - .envs(envs) - .args(&command_args[1..]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(cmd, UV_PATH.as_str()).await - } + let mut cmd = Command::new(command_args[0]); + cmd.env_clear() + .envs(PROXY_ENVS.clone()) + .envs(envs) + .args(&command_args[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(cmd, UV_PATH.as_str()).await } #[cfg(windows)] @@ -1853,8 +1695,6 @@ pub async fn handle_python_reqs( worker_dir: &str, _occupancy_metrics: &mut Option<&mut OccupancyMetrics>, py_version: PyVersion, - // TODO: Remove (Deprecated) - mut no_uv_install: bool, ) -> error::Result> { let worker_dir = worker_dir.to_string(); @@ -1906,19 +1746,10 @@ pub async fn handle_python_reqs( .await; // Drop lock, so next print success can fire } - no_uv_install |= *USE_PIP_INSTALL; - if no_uv_install { - append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; - tracing::warn!("Fallback to pip"); - } // Parallelism level (N) - let parallel_limit = if no_uv_install { - 1 - } else { - // Semaphore will panic if value less then 1 - PY_CONCURRENT_DOWNLOADS.clamp(1, 30) - }; + let parallel_limit = // Semaphore will panic if value less then 1 + PY_CONCURRENT_DOWNLOADS.clamp(1, 30); tracing::info!( workspace_id = %w_id, @@ -1953,11 +1784,7 @@ pub async fn handle_python_reqs( if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - let py_prefix = if no_uv_install { - PIP_CACHE_DIR - } else { - &py_version.to_cache_dir() - }; + let py_prefix = &py_version.to_cache_dir(); let venv_p = format!( "{py_prefix}/{}", @@ -2097,13 +1924,7 @@ pub async fn handle_python_reqs( let mut req_tl = 0; if total_to_install > 0 { let mut logs = String::new(); - // Do we use UV? - if no_uv_install { - logs.push_str("\n\n--- PIP INSTALL ---\n"); - } else { - logs.push_str("\n\n--- UV PIP INSTALL ---\n"); - } - + logs.push_str("\n\n--- UV PIP INSTALL ---\n"); logs.push_str("\nTo be installed: \n\n"); for (req, _) in &req_with_penv { if req.len() > req_tl { @@ -2135,13 +1956,9 @@ pub async fn handle_python_reqs( let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro); let total_time = std::time::Instant::now(); - let py_path = if no_uv_install { - None - } else { - py_version - .get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics) - .await? - }; + let py_path = py_version + .get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics) + .await?; let has_work = req_with_penv.len() > 0; for ((i, (req, venv_p)), mut kill_rx) in @@ -2196,7 +2013,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level()) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -2242,7 +2059,6 @@ pub async fn handle_python_reqs( &job_dir, pip_indexes, py_path, - no_uv_install, &worker_dir ).await { Ok(r) => r, @@ -2348,7 +2164,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level())); } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 590b57227c..5e647dd7ce 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -264,10 +264,6 @@ pub async fn create_token_for_owner( pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); -pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); -// Used as fallback now -pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); - pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); @@ -281,7 +277,6 @@ pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313" pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); -pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); @@ -300,7 +295,6 @@ const NUM_SECS_PING: u64 = 5; const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); -const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh"); pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days @@ -833,13 +827,6 @@ pub async fn run_worker( "download_deps.py.sh", INCLUDE_DEPS_PY_SH_CONTENT, ); - - // TODO: Remove (Deprecated) - let _ = write_file( - &worker_dir, - "download_deps.py.pip.sh", - INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK, - ); } let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 03ba19d2ad..6c7ed70dfc 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -39,8 +39,7 @@ use crate::csharp_executor::generate_nuget_lockfile; use crate::php_executor::{composer_install, parse_php_imports}; #[cfg(feature = "python")] use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, USE_PIP_COMPILE, - USE_PIP_INSTALL, + create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, }; #[cfg(feature = "rust")] use crate::rust_executor::generate_cargo_lockfile; @@ -1602,8 +1601,6 @@ async fn python_dep( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, annotated_pyv_numeric: Option, annotations: PythonAnnotations, - no_uv_compile: bool, - no_uv_install: bool, ) -> std::result::Result { create_dependencies_dir(job_dir).await; @@ -1634,7 +1631,6 @@ async fn python_dep( occupancy_metrics, final_version, annotations.no_cache, - no_uv_compile, ) .await; // install the dependencies to pre-fill the cache @@ -1651,7 +1647,6 @@ async fn python_dep( worker_dir, occupancy_metrics, final_version, - no_uv_install, ) .await; @@ -1716,21 +1711,6 @@ async fn capture_dependency_job( .await? .join("\n") }; - let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = anns; - if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { - if let Err(e) = sqlx::query!( - r#" - INSERT INTO metrics (id, value) - VALUES ('no_uv_usage_py', $1) - "#, - serde_json::to_value("").map_err(to_anyhow)? - ) - .execute(db) - .await - { - tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e); - } - } python_dep( reqs, @@ -1745,8 +1725,6 @@ async fn capture_dependency_job( &mut Some(occupancy_metrics), annotated_pyv_numeric, anns, - no_uv_compile | no_uv, - no_uv_install | no_uv, ) .await } @@ -1767,21 +1745,6 @@ async fn capture_dependency_job( let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); - if *USE_PIP_COMPILE || *USE_PIP_INSTALL { - if let Err(e) = sqlx::query!( - r#" - INSERT INTO metrics (id, value) - VALUES ('no_uv_usage_ansible', $1) - "#, - serde_json::to_value("").map_err(to_anyhow)? - ) - .execute(db) - .await - { - tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e); - }; - } - python_dep( reqs, job_id, @@ -1795,8 +1758,6 @@ async fn capture_dependency_job( &mut Some(occupancy_metrics), None, PythonAnnotations::default(), - false, - false, ) .await }