From 097d9288c58076882f1991e2fb33e4441fe332d3 Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Wed, 11 Feb 2026 12:18:51 +0100 Subject: [PATCH] feat(uv): index resolve strategy (#7885) * fix * fix Signed-off-by: pyranota * Update backend/windmill-worker/src/python_executor.rs Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * Update backend/windmill-worker/src/python_executor.rs Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --------- Signed-off-by: pyranota Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --- backend/src/main.rs | 7 ++--- backend/src/monitor.rs | 14 +++++++++- .../windmill-common/src/global_settings.rs | 1 + .../nsjail/download_deps.py.sh | 15 +++++------ .../windmill-worker/src/python_executor.rs | 24 +++++++++-------- backend/windmill-worker/src/worker.rs | 1 + .../src/lib/components/InstanceSetting.svelte | 2 +- .../src/lib/components/instanceSettings.ts | 26 +++++++++++++++++++ 8 files changed, 66 insertions(+), 24 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 4a7b23aab7..083575e418 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -46,8 +46,8 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, UV_INDEX_STRATEGY_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, @@ -103,7 +103,7 @@ use crate::monitor::{ reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, - reload_worker_config, MonitorIteration, + reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; #[cfg(feature = "parquet")] @@ -1564,6 +1564,7 @@ async fn process_notify_event( SCIM_TOKEN_SETTING => reload_scim_token_setting(conn).await, EXTRA_PIP_INDEX_URL_SETTING => reload_extra_pip_index_url_setting(conn).await, PIP_INDEX_URL_SETTING => reload_pip_index_url_setting(conn).await, + UV_INDEX_STRATEGY_SETTING => reload_uv_index_strategy_setting(conn).await, INSTANCE_PYTHON_VERSION_SETTING => { reload_instance_python_version_setting(conn).await } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index b854516b69..09cc8ad66d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -57,6 +57,7 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + UV_INDEX_STRATEGY_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, @@ -88,7 +89,7 @@ use windmill_worker::{ SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, - POWERSHELL_REPO_URL, + POWERSHELL_REPO_URL, UV_INDEX_STRATEGY, }; #[cfg(feature = "parquet")] @@ -317,6 +318,7 @@ pub async fn initial_load( reload_job_default_timeout_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; + reload_uv_index_strategy_setting(&conn).await; reload_npm_config_registry_setting(&conn).await; reload_bunfig_install_scopes_setting(&conn).await; reload_instance_python_version_setting(&conn).await; @@ -1250,6 +1252,16 @@ pub async fn reload_pip_index_url_setting(conn: &Connection) { .await; } +pub async fn reload_uv_index_strategy_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + UV_INDEX_STRATEGY_SETTING, + "UV_INDEX_STRATEGY", + UV_INDEX_STRATEGY.clone(), + ) + .await; +} + pub async fn reload_instance_python_version_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 59a941701b..f1a1613ed2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -19,6 +19,7 @@ pub const RUBY_REPOS_SETTING: &str = "ruby_repos"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; +pub const UV_INDEX_STRATEGY_SETTING: &str = "uv_index_strategy"; pub const INSTANCE_PYTHON_VERSION_SETTING: &str = "instance_python_version"; pub const SCIM_TOKEN_SETTING: &str = "scim_token"; pub const SAML_METADATA_SETTING: &str = "saml_metadata"; diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 06a61ac19f..ce056e7e76 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -19,17 +19,16 @@ then echo "\$TRUSTED_HOST is set to $TRUSTED_HOST" fi -CMD="/usr/local/bin/uv pip install -\"$REQ\" ---target \"$TARGET\" ---no-cache +CMD="/usr/local/bin/uv pip install +\"$REQ\" +--target \"$TARGET\" +--no-cache --no-config ---no-color ---no-deps +--no-color +--no-deps --link-mode=copy -$PY_PATH +$PY_PATH $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG ---index-strategy unsafe-best-match --system --reinstall " diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 966a76d90d..8dc689a3c2 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -135,6 +135,7 @@ use crate::{ worker_utils::ping_job_status, PyV, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -221,6 +222,9 @@ pub async fn uv_pip_compile( requirements.to_string() }; + let uv_index_strategy = UV_INDEX_STRATEGY.read().await.clone(); + let uv_index_strategy = uv_index_strategy.as_deref().unwrap_or("unsafe-best-match"); + let py_version_str = py_version.clone().to_string(); // Include python version to requirements.in // We need it because same hash based on requirements.in can get calculated even for different python versions @@ -230,7 +234,7 @@ pub async fn uv_pip_compile( #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?; - let req_hash = format!("py-{}", calculate_hash(&requirements)); + let req_hash = format!("py-{}-{uv_index_strategy}", calculate_hash(&requirements)); if !no_cache { if let Some(db) = conn.as_sql() { @@ -271,11 +275,6 @@ pub async fn uv_pip_compile( "--strip-extras", "-o", "requirements.txt", - // Prefer main index over extra - // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes - // TODO: Use env variable that can be toggled from UI - "--index-strategy", - "unsafe-best-match", // Target to /tmp/windmill/cache/uv "--cache-dir", UV_CACHE_DIR, @@ -321,6 +320,7 @@ pub async fn uv_pip_compile( #[cfg(unix)] let uv_cmd = UV_PATH.as_str(); + let mut child_cmd = Command::new(uv_cmd); child_cmd .current_dir(job_dir) @@ -328,6 +328,7 @@ pub async fn uv_pip_compile( .env("HOME", HOME_ENV.to_string()) .env("PATH", PATH_ENV.to_string()) .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string()) + .env("UV_INDEX_STRATEGY", uv_index_strategy) .envs(PROXY_ENVS.clone()) .args(&args) .stdout(Stdio::piped()) @@ -1353,6 +1354,9 @@ async fn spawn_uv_install( py_path: Option, worker_dir: &str, ) -> Result, Error> { + let uv_index_strategy_guard = UV_INDEX_STRATEGY.read().await.clone(); + let uv_index_strategy = uv_index_strategy_guard.as_deref().unwrap_or("unsafe-best-match"); + if !*DISABLE_NSJAIL { tracing::info!( workspace_id = %w_id, @@ -1375,6 +1379,7 @@ async fn spawn_uv_install( if *NATIVE_CERT { vars.push(("UV_NATIVE_TLS", "true")); } + let _owner; if let Some(py_path) = py_path.as_ref() { _owner = format!( @@ -1385,6 +1390,7 @@ async fn spawn_uv_install( } vars.push(("REQ", &req)); vars.push(("TARGET", venv_p)); + vars.push(("UV_INDEX_STRATEGY", uv_index_strategy)); std::fs::create_dir_all(venv_p)?; let nsjail_proto = format!("{req}.config.proto"); @@ -1430,11 +1436,6 @@ async fn spawn_uv_install( "--no-config", "--link-mode=copy", "--system", - // Prefer main index over extra - // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes - // TODO: Use env variable that can be toggled from UI - "--index-strategy", - "unsafe-best-match", "--target", venv_p, "--no-cache", @@ -1464,6 +1465,7 @@ async fn spawn_uv_install( let mut envs = vec![("PATH", PATH_ENV.as_str())]; envs.push(("HOME", HOME_ENV.as_str())); + envs.push(("UV_INDEX_STRATEGY", uv_index_strategy)); if let Some(url) = pip_index_url.as_ref() { command_args.extend(["--index-url", url]); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 06f59e3814..fad2878477 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -584,6 +584,7 @@ lazy_static::lazy_static! { pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref UV_INDEX_STRATEGY: Arc>> = Arc::new(RwLock::new(None)); pub static ref INSTANCE_PYTHON_VERSION: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 0be0c05891..fe165fa636 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -50,7 +50,7 @@ (setting.fieldType == 'select' || setting.fieldType == 'select_python') && $values[setting.key] == undefined ) { - $values[setting.key] = 'default' + $values[setting.key] = setting.defaultValue ? setting.defaultValue() : 'default' } let latestKeyRenewalAttempt: { diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 1b29688316..c722815fe0 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -329,6 +329,32 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'UV index strategy', + description: + 'Strategy for resolving packages from multiple indexes. See uv docs', + key: 'uv_index_strategy', + fieldType: 'select', + placeholder: 'unsafe-best-match', + defaultValue: () => "unsafe-best-match", + select_items: [ + { + label: 'first-index', + tooltip: 'Only use the first index that contains the package' + }, + { + label: 'unsafe-first-match', + tooltip: 'Search for packages across all indexes, preferring the first match' + }, + { + label: 'unsafe-best-match (default)', + value: 'unsafe-best-match', + tooltip: 'Search for packages across all indexes, preferring the best match' + } + ], + storage: 'setting', + ee_only: '' + }, { label: 'Npm config registry', description: 'Add private npm registry',