kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: gracefully handle uninitialized OTEL tracing proxy port (#8274)

Browse Source 
* fix: gracefully handle uninitialized OTEL tracing proxy port

When OTEL tracing proxy is enabled but the MITM proxy port hasn't been
assigned yet (race condition at startup, or NUM_WORKERS > 1), fall back
to standard proxy envs instead of failing the job with
"OTEL tracing proxy port not initialized".

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: log to job logs when OTEL tracing proxy is unavailable

When the OTEL tracing proxy is enabled but the port isn't initialized
(race at startup or NUM_WORKERS > 1), append a warning to the job logs
explaining why HTTP request tracing is unavailable for that job.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-03-09 09:34:21 +00:00
committed by GitHub
parent f7afcdb704
commit 6636921609
10 changed files with 83 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/windmill-worker/src/bash_executor.rs
View File

@@ -215,7 +215,10 @@ exit $exit_status
.current_dir(job_dir)
.env_clear()
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.args(cmd_args)
@@ -241,7 +244,10 @@ exit $exit_status
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Bash).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bash, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())

14
backend/windmill-worker/src/bun_executor.rs
View File

@@ -1564,7 +1564,9 @@ try {{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn).await?,
)
.envs(common_bun_proc_envs)
.env("PATH", PATH_ENV.as_str())
.args(args)
@@ -1582,7 +1584,10 @@ try {{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn)
.await?,
)
.envs(common_bun_proc_envs)
.stdin(Stdio::null())
.stdout(Stdio::piped())
@@ -1613,7 +1618,10 @@ try {{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Bun).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Bun, &job.id, &job.workspace_id, conn)
.await?,
)
.envs(common_bun_proc_envs)
.stdin(Stdio::null())
.stdout(Stdio::piped())

10
backend/windmill-worker/src/csharp_executor.rs
View File

@@ -600,7 +600,10 @@ pub async fn handle_csharp_job(
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -633,7 +636,10 @@ pub async fn handle_csharp_job(
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::CSharp).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::CSharp, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)

12
backend/windmill-worker/src/deno_executor.rs
View File

@@ -121,11 +121,13 @@ async fn get_common_deno_proc_envs(
}
// Add proxy envs (including OTEL tracing proxy if enabled for deno)
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno)
.await
.unwrap_or_default()
{
deno_envs.insert(k.to_string(), v);
if let Some(conn) = conn {
for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno, job_id, w_id, conn)
.await
.unwrap_or_default()
{
deno_envs.insert(k.to_string(), v);
}
}
return deno_envs;

4
backend/windmill-worker/src/go_executor.rs
View File

@@ -354,7 +354,7 @@ func Run(req Req) (interface{{}}, error){{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -375,7 +375,7 @@ func Run(req Req) (interface{{}}, error){{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go).await?)
.envs(get_proxy_envs_for_lang(&ScriptLang::Go, &job.id, &job.workspace_id, conn).await?)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)

4
backend/windmill-worker/src/nu_executor.rs
View File

@@ -264,7 +264,7 @@ async fn run<'a>(
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
.args(vec![
"--config",
"run.config.proto",
@@ -303,7 +303,7 @@ async fn run<'a>(
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu).await?)
.envs(get_proxy_envs_for_lang(&ScriptLang::Nu, &job.id, &job.workspace_id, conn).await?)
// TODO(v1):
// "--plugins",
// &format!(

10
backend/windmill-worker/src/python_executor.rs
View File

@@ -841,7 +841,10 @@ mount {{
.env_clear()
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -867,7 +870,10 @@ mount {{
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Python3).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Python3, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)

10
backend/windmill-worker/src/ruby_executor.rs
View File

@@ -812,7 +812,10 @@ mount {{
.envs(envs)
.envs(reserved_variables)
.envs(RUBY_PROXY_ENVS.clone())
.envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
.await?,
)
.args(vec![
"--config",
"run.config.proto",
@@ -851,7 +854,10 @@ mount {{
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(reserved_variables)
.envs(RUBY_PROXY_ENVS.clone())
.envs(get_proxy_envs_for_lang(&ScriptLang::Ruby).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Ruby, &job.id, &job.workspace_id, conn)
.await?,
)
.envs(envs);
cmd.stdin(Stdio::null())

10
backend/windmill-worker/src/rust_executor.rs
View File

@@ -700,7 +700,10 @@ pub async fn handle_rust_job(
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -716,7 +719,10 @@ pub async fn handle_rust_job(
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(get_proxy_envs_for_lang(&ScriptLang::Rust).await?)
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rust, &job.id, &job.workspace_id, conn)
.await?,
)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)

26
backend/windmill-worker/src/worker.rs
View File

@@ -745,21 +745,37 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool {
/// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment.
pub async fn get_proxy_envs_for_lang(
lang: &ScriptLang,
job_id: &uuid::Uuid,
w_id: &str,
conn: &Connection,
) -> anyhow::Result<Vec<(&'static str, String)>> {
#[cfg(all(feature = "private", feature = "enterprise"))]
if is_otel_tracing_proxy_enabled_for_lang(lang).await {
return get_otel_tracing_proxy_envs().await;
return get_otel_tracing_proxy_envs(job_id, w_id, conn).await;
}
let _ = lang;
let _ = (lang, job_id, w_id, conn);
Ok(PROXY_ENVS.clone())
}
#[cfg(all(feature = "private", feature = "enterprise"))]
async fn get_otel_tracing_proxy_envs() -> anyhow::Result<Vec<(&'static str, String)>> {
let port = crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
async fn get_otel_tracing_proxy_envs(
job_id: &uuid::Uuid,
w_id: &str,
conn: &Connection,
) -> anyhow::Result<Vec<(&'static str, String)>> {
let port = match *crate::otel_tracing_proxy_ee::TRACING_PROXY_PORT
.read()
.await
.ok_or_else(|| anyhow::anyhow!("OTEL tracing proxy port not initialized"))?;
{
Some(p) => p,
None => {
let reason = "OTEL tracing proxy is enabled but not available (not initialized yet, or NUM_WORKERS > 1). \
This job's HTTP requests will not be traced.";
tracing::warn!("{}", reason);
append_logs(job_id, w_id, format!("\n[warning] {reason}\n"), conn).await;
return Ok(PROXY_ENVS.clone());
}
};
let proxy_url = format!("http://127.0.0.1:{}", port);
Ok(vec![
("HTTP_PROXY", proxy_url.clone()),