kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use process groups to improve zombie job handling (#6157)

Browse Source 
* useProcessGroups

* nit
This commit is contained in:
Ruben Fiszel
2025-07-10 10:49:56 +02:00
committed by GitHub
parent 94f016768d
commit 05860e5396
10 changed files with 110 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
backend/windmill-worker/src/python_executor.rs
View File

@@ -38,6 +38,8 @@ use windmill_common::variables::get_secret_value_as_admin;
use std::env::var;
use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo};
use process_wrap::tokio::TokioChildWrapper;
lazy_static::lazy_static! {
pub(crate) static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
tracing::warn!("PYTHON_PATH is set to {} and thus python will not be managed by uv and stay static regardless of annotation and instance settings. NOT RECOMMENDED", v);
@@ -88,7 +90,8 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver<Pipt
while let Some(task) = rx.recv().await {
if let Some(os) = get_object_store().await {
match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await {
match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await
{
Ok(()) => {
tracing::info!("Successfully uploaded piptar for {}", task.venv_path);
}
@@ -97,7 +100,10 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver<Pipt
}
}
} else {
tracing::warn!("S3 object store not available for piptar upload: {}", task.venv_path);
tracing::warn!(
"S3 object store not available for piptar upload: {}",
task.venv_path
);
}
}
}
@@ -336,26 +342,30 @@ pub async fn uv_pip_compile(
)
.env(
"APPDATA",
std::env::var("APPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())),
std::env::var("APPDATA").unwrap_or_else(|_| {
format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())
}),
)
.env(
"ComSpec",
std::env::var("ComSpec").unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
std::env::var("ComSpec")
.unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
)
.env(
"PATHEXT",
std::env::var("PATHEXT").unwrap_or_else(|_|
std::env::var("PATHEXT").unwrap_or_else(|_| {
String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
),
}),
)
.env(
"ProgramData",
std::env::var("ProgramData").unwrap_or_else(|_| String::from("C:\\ProgramData")),
std::env::var("ProgramData")
.unwrap_or_else(|_| String::from("C:\\ProgramData")),
)
.env(
"ProgramFiles",
std::env::var("ProgramFiles").unwrap_or_else(|_| String::from("C:\\Program Files")),
std::env::var("ProgramFiles")
.unwrap_or_else(|_| String::from("C:\\Program Files")),
);
}
@@ -1290,7 +1300,7 @@ async fn spawn_uv_install(
// If none, it is system python
py_path: Option<String>,
worker_dir: &str,
) -> Result<tokio::process::Child, Error> {
) -> Result<Box<dyn TokioChildWrapper>, Error> {
if !*DISABLE_NSJAIL {
tracing::info!(
workspace_id = %w_id,
@@ -1453,26 +1463,30 @@ async fn spawn_uv_install(
)
.env(
"APPDATA",
std::env::var("APPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())),
std::env::var("APPDATA").unwrap_or_else(|_| {
format!("{}\\AppData\\Roaming", crate::USERPROFILE_ENV.as_str())
}),
)
.env(
"ComSpec",
std::env::var("ComSpec").unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
std::env::var("ComSpec")
.unwrap_or_else(|_| String::from("C:\\Windows\\System32\\cmd.exe")),
)
.env(
"PATHEXT",
std::env::var("PATHEXT").unwrap_or_else(|_|
std::env::var("PATHEXT").unwrap_or_else(|_| {
String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
),
}),
)
.env(
"ProgramData",
std::env::var("ProgramData").unwrap_or_else(|_| String::from("C:\\ProgramData")),
std::env::var("ProgramData")
.unwrap_or_else(|_| String::from("C:\\ProgramData")),
)
.env(
"ProgramFiles",
std::env::var("ProgramFiles").unwrap_or_else(|_| String::from("C:\\Program Files")),
std::env::var("ProgramFiles")
.unwrap_or_else(|_| String::from("C:\\Program Files")),
)
.args(&command_args[1..])
.stdout(Stdio::piped())
@@ -1901,7 +1915,7 @@ pub async fn handle_python_reqs(
let mut stderr_buf = String::new();
let mut stderr_pipe = uv_install_proccess
.stderr
.stderr()
.take()
.ok_or(anyhow!("Cannot take stderr from uv_install_proccess"))?;
let stderr_future = stderr_pipe.read_to_string(&mut stderr_buf);
@@ -1918,14 +1932,14 @@ pub async fn handle_python_reqs(
tokio::select! {
// Canceled
_ = kill_rx.recv() => {
uv_install_proccess.kill().await?;
Box::into_pin(uv_install_proccess.kill()).await?;
pids.lock().await.get_mut(i).and_then(|e| e.take());
return Err(anyhow::anyhow!("uv pip install was canceled"));
},
(_, exitstatus) = async {
// See tokio::process::Child::wait_with_output() for more context
// Sometimes uv_install_proccess.wait() is not exiting if stderr is not awaited before it :/
(stderr_future.await, uv_install_proccess.wait().await)
(stderr_future.await, Box::into_pin(uv_install_proccess.wait()).await)
} => match exitstatus {
Ok(status) => if !status.success() {
tracing::warn!(
@@ -2294,4 +2308,3 @@ for line in sys.stdin:
)
.await
}