feat: Replace pip-compile with uv (#4460)
* Update `shell.nix`
- Replace pip-compile with uv packages
- Pin rust version
- Add var to trigger windmill print more info in stdout
* Replace `pip-compile` with `uv` (dirty + untested)
* Fix arguments passed to uv
Some of the flags are included by default in UV and can be safely removed:
- --resolver=backtracking
- --no-emit-index-url
Also uv does not support `--pip-args` and suggests to directly pass args to uv.
* Remove extra `dbg!`
* Replace 'pip-compile' with 'uv' in Dockerfile
* Add fallback option to `pip-compile` (Disabled)
* Add `uv` to `docker/DockerfileSlim*`
* Add `get_annotation_python` and rename `get_annotation` to `get_annotation_ts`
* Add option to fallback to pip-compile
Put `# no_uv` on top the file for specific python script
Or set `USE_PIP_COMPILE` variable to `true`
* Put back `pip-tools` into shell.nix
* Make sure lockfile resolves again if `#no_uv` used
Add #no_uv to the end of requirements (requirements.in)
That way if something breaks for customer, then they put #no_uv and new lockfile will be resolved
* Put `pip install pip-tools` in original spot
* Fix compilation error
* Fix EE compilation error
error[E0658]: attributes on expressions are experimental
--> windmill-worker/src/python_executor.rs:144:5
|
144 | #[cfg(feature = "enterprise")]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
= note: see issue #15701 <https://github.com/rust-lang/rust/issues/15701> for more information
* Add `no_cache` annotation
Will force recalculation of lockfile
And block uv from using cached values
* Target uv cache to /tmp/windmill/cache
* Prohibit uv from managing python
* Add uv to DockerfileBackendTests
* Pin uv version to 0.4.18 in Dockerfiles
* Dont put `#no_uv` in requirements.in
Instead postfix hash for requirements.in with `-no_uv`
* Push Warning to logs if fallbacked to pip-compile
---------
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
pyranota
@@ -36,6 +36,9 @@ lazy_static::lazy_static! {
|
||||
static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
|
||||
static ref PIP_INDEX_CERT: Option<String> = std::env::var("PIP_INDEX_CERT").ok();
|
||||
|
||||
static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
|
||||
.ok().map(|flag| flag == "true").unwrap_or(false);
|
||||
|
||||
|
||||
static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
|
||||
|
||||
@@ -61,7 +64,7 @@ use crate::{
|
||||
handle_child::handle_child,
|
||||
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
|
||||
LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
|
||||
PIP_INDEX_URL, TZ_ENV,
|
||||
PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR,
|
||||
};
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -96,7 +99,7 @@ pub fn handle_ephemeral_token(x: String) -> String {
|
||||
x
|
||||
}
|
||||
|
||||
pub async fn pip_compile(
|
||||
pub async fn uv_pip_compile(
|
||||
job_id: &Uuid,
|
||||
requirements: &str,
|
||||
mem_peak: &mut i32,
|
||||
@@ -106,6 +109,10 @@ pub async fn pip_compile(
|
||||
worker_name: &str,
|
||||
w_id: &str,
|
||||
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
|
||||
// Fallback to pip-compile. Will be removed in future
|
||||
mut no_uv: bool,
|
||||
// Debug-only flag
|
||||
no_cache: bool,
|
||||
) -> error::Result<String> {
|
||||
let mut logs = String::new();
|
||||
logs.push_str(&format!("\nresolving dependencies..."));
|
||||
@@ -142,83 +149,178 @@ pub async fn pip_compile(
|
||||
#[cfg(feature = "enterprise")]
|
||||
let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?;
|
||||
|
||||
let req_hash = format!("py-{}", calculate_hash(&requirements));
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
req_hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
logs.push_str(&format!("\nfound cached resolution: {req_hash}"));
|
||||
return Ok(cached);
|
||||
let mut req_hash = format!("py-{}", calculate_hash(&requirements));
|
||||
|
||||
if no_uv || *USE_PIP_COMPILE {
|
||||
logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)"));
|
||||
// Set no_uv if not setted
|
||||
no_uv = true;
|
||||
// Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable.
|
||||
// Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db).
|
||||
// It will recalculate lockfile even if inputs have not been changed.
|
||||
req_hash.push_str("-no_uv");
|
||||
// Will be in format:
|
||||
// py-000..000-no_uv
|
||||
}
|
||||
if !no_cache {
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
req_hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
logs.push_str(&format!("\nfound cached resolution: {req_hash}"));
|
||||
return Ok(cached);
|
||||
}
|
||||
}
|
||||
let file = "requirements.in";
|
||||
|
||||
write_file(job_dir, file, &requirements)?;
|
||||
|
||||
let mut args = vec![
|
||||
"-q",
|
||||
"--no-header",
|
||||
file,
|
||||
"--resolver=backtracking",
|
||||
"--strip-extras",
|
||||
];
|
||||
let mut pip_args = vec![];
|
||||
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_extra_index_url.as_ref() {
|
||||
args.extend(["--extra-index-url", url, "--no-emit-index-url"]);
|
||||
pip_args.push(format!("--extra-index-url {}", url));
|
||||
}
|
||||
let pip_index_url = PIP_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_index_url.as_ref() {
|
||||
args.extend(["--index-url", url, "--no-emit-index-url"]);
|
||||
pip_args.push(format!("--index-url {}", url));
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
args.extend(["--trusted-host", host]);
|
||||
}
|
||||
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
|
||||
args.extend(["--cert", cert_path]);
|
||||
}
|
||||
let pip_args_str = pip_args.join(" ");
|
||||
if pip_args.len() > 0 {
|
||||
args.extend(["--pip-args", &pip_args_str]);
|
||||
}
|
||||
tracing::debug!("pip-compile args: {:?}", args);
|
||||
// Fallback pip-compile. Will be removed in future
|
||||
if no_uv {
|
||||
tracing::debug!("Fallback to pip-compile");
|
||||
|
||||
let mut args = vec![
|
||||
"-q",
|
||||
"--no-header",
|
||||
file,
|
||||
"--resolver=backtracking",
|
||||
"--strip-extras",
|
||||
];
|
||||
let mut pip_args = vec![];
|
||||
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_extra_index_url.as_ref() {
|
||||
args.extend(["--extra-index-url", url, "--no-emit-index-url"]);
|
||||
pip_args.push(format!("--extra-index-url {}", url));
|
||||
}
|
||||
let pip_index_url = PIP_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_index_url.as_ref() {
|
||||
args.extend(["--index-url", url, "--no-emit-index-url"]);
|
||||
pip_args.push(format!("--index-url {}", url));
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
args.extend(["--trusted-host", host]);
|
||||
}
|
||||
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
|
||||
args.extend(["--cert", cert_path]);
|
||||
}
|
||||
let pip_args_str = pip_args.join(" ");
|
||||
if pip_args.len() > 0 {
|
||||
args.extend(["--pip-args", &pip_args_str]);
|
||||
}
|
||||
tracing::debug!("pip-compile args: {:?}", args);
|
||||
|
||||
let mut child_cmd = Command::new("pip-compile");
|
||||
child_cmd
|
||||
.current_dir(job_dir)
|
||||
.args(args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
let child_process = start_child_process(child_cmd, "pip-compile").await?;
|
||||
append_logs(&job_id, &w_id, logs, db).await;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
"pip-compile",
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
} else {
|
||||
let mut args = vec![
|
||||
"pip",
|
||||
"compile",
|
||||
"-q",
|
||||
"--no-header",
|
||||
file,
|
||||
"--strip-extras",
|
||||
"-o",
|
||||
"requirements.txt",
|
||||
// Prefer main index over extra
|
||||
// https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
|
||||
// TODO: Use env variable that can be toggled from UI
|
||||
"--index-strategy",
|
||||
"unsafe-best-match",
|
||||
// Target to /tmp/windmill/cache/uv
|
||||
"--cache-dir",
|
||||
UV_CACHE_DIR,
|
||||
// We dont want UV to manage python installations
|
||||
"--python-preference",
|
||||
"only-system",
|
||||
"--no-python-downloads",
|
||||
];
|
||||
if no_cache {
|
||||
args.extend(["--no-cache"]);
|
||||
}
|
||||
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_extra_index_url.as_ref() {
|
||||
args.extend(["--extra-index-url", url]);
|
||||
}
|
||||
let pip_index_url = PIP_INDEX_URL
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.map(handle_ephemeral_token);
|
||||
if let Some(url) = pip_index_url.as_ref() {
|
||||
args.extend(["--index-url", url]);
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
args.extend(["--trusted-host", host]);
|
||||
}
|
||||
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
|
||||
args.extend(["--cert", cert_path]);
|
||||
}
|
||||
tracing::debug!("uv args: {:?}", args);
|
||||
|
||||
let mut child_cmd = Command::new("uv");
|
||||
child_cmd
|
||||
.current_dir(job_dir)
|
||||
.args(args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
let child_process = start_child_process(child_cmd, "uv").await?;
|
||||
append_logs(&job_id, &w_id, logs, db).await;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
// TODO: Rename to uv-pip-compile?
|
||||
"uv",
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
}
|
||||
|
||||
let mut child_cmd = Command::new("pip-compile");
|
||||
child_cmd
|
||||
.current_dir(job_dir)
|
||||
.args(args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
let child_process = start_child_process(child_cmd, "pip-compile").await?;
|
||||
append_logs(&job_id, &w_id, logs, db).await;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
child_process,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
"pip-compile",
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
let path_lock = format!("{job_dir}/requirements.txt");
|
||||
let mut file = File::open(path_lock).await?;
|
||||
let mut req_content = "".to_string();
|
||||
@@ -784,6 +886,7 @@ async fn handle_python_deps(
|
||||
let requirements = match requirements_o {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
let annotation = windmill_common::worker::get_annotation_python(inner_content);
|
||||
let mut already_visited = vec![];
|
||||
|
||||
let requirements = windmill_parser_py_imports::parse_python_imports(
|
||||
@@ -798,7 +901,7 @@ async fn handle_python_deps(
|
||||
if requirements.is_empty() {
|
||||
"".to_string()
|
||||
} else {
|
||||
pip_compile(
|
||||
uv_pip_compile(
|
||||
job_id,
|
||||
&requirements,
|
||||
mem_peak,
|
||||
@@ -808,6 +911,8 @@ async fn handle_python_deps(
|
||||
worker_name,
|
||||
w_id,
|
||||
occupancy_metrics,
|
||||
annotation.no_uv,
|
||||
annotation.no_cache,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
|
||||
Reference in New Issue
Block a user