feat: Add OTel auto-instrumentation EE implementation (otel_auto_instrumentation_impl.rs)

- Create full EE implementation with config caching from global_settings
- Add environment variable generation for Python and TypeScript scripts
- Add OTLP JSON trace parsing with resource/span attribute extraction
- Add built-in HTTP collector server using Axum
- Add database storage for spans in job_otel_traces table
- Include comprehensive unit tests
- Update imports in bun_executor, deno_executor, python_executor
- Update OSS stub to re-export from impl module when enterprise feature enabled

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Ruben Fiszel <rubenfiszel@users.noreply.github.com>

backend/Cargo.lock

@@ -15814,6 +15814,7 @@ dependencies = [
  "aws-credential-types",
  "aws-sdk-bedrockruntime",
  "aws-smithy-types",
+ "axum",
  "backon",
  "base64 0.22.1",
  "bit-vec 0.6.3",
@@ -15863,6 +15864,7 @@ dependencies = [
  "postgres-native-tls 0.5.1",
  "process-wrap",
  "prometheus",
+ "prost",
  "rand 0.9.0",
  "regex",
  "reqwest 0.12.24",

backend/Cargo.toml

@@ -450,3 +450,4 @@ oracle = { version = "0.6.3", features = ["chrono"] }
 rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
 strum = { version = "0.27", features = ["derive"] }
 strum_macros = "^0"
+prost = "0.13.5"

backend/migrations/20260107000000_job_otel_traces.down.sql

@@ -0,0 +1,9 @@
+-- Drop indexes
+DROP INDEX IF EXISTS idx_job_otel_traces_job_workspace;
+DROP INDEX IF EXISTS idx_job_otel_traces_created_at;
+DROP INDEX IF EXISTS idx_job_otel_traces_trace_id;
+DROP INDEX IF EXISTS idx_job_otel_traces_workspace_id;
+DROP INDEX IF EXISTS idx_job_otel_traces_job_id;
+
+-- Drop table
+DROP TABLE IF EXISTS job_otel_traces;

backend/migrations/20260107000000_job_otel_traces.up.sql

@@ -0,0 +1,28 @@
+-- Add table to store OTel traces from auto-instrumented scripts
+CREATE TABLE IF NOT EXISTS job_otel_traces (
+    id BIGSERIAL PRIMARY KEY,
+    job_id UUID NOT NULL,
+    workspace_id VARCHAR(50) NOT NULL,
+    trace_id VARCHAR(32) NOT NULL,
+    span_id VARCHAR(16) NOT NULL,
+    parent_span_id VARCHAR(16),
+    operation_name VARCHAR(255) NOT NULL,
+    service_name VARCHAR(255),
+    start_time_unix_nano BIGINT NOT NULL,
+    end_time_unix_nano BIGINT NOT NULL,
+    duration_ns BIGINT GENERATED ALWAYS AS (end_time_unix_nano - start_time_unix_nano) STORED,
+    status_code SMALLINT DEFAULT 0,
+    status_message TEXT,
+    attributes JSONB DEFAULT '{}',
+    events JSONB DEFAULT '[]',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Create indexes for efficient querying
+CREATE INDEX IF NOT EXISTS idx_job_otel_traces_job_id ON job_otel_traces(job_id);
+CREATE INDEX IF NOT EXISTS idx_job_otel_traces_workspace_id ON job_otel_traces(workspace_id);
+CREATE INDEX IF NOT EXISTS idx_job_otel_traces_trace_id ON job_otel_traces(trace_id);
+CREATE INDEX IF NOT EXISTS idx_job_otel_traces_created_at ON job_otel_traces(created_at);
+
+-- Composite index for common query patterns
+CREATE INDEX IF NOT EXISTS idx_job_otel_traces_job_workspace ON job_otel_traces(job_id, workspace_id);

backend/windmill-api/src/jobs.rs

@@ -373,6 +373,7 @@ pub fn workspace_unauthed_service() -> Router {
             get(get_completed_job_logs_tail),
         )
         .route("/get_args/:id", get(get_args))
+        .route("/get_otel_traces/:id", get(get_job_otel_traces))
         .route("/queue/get_started_at_by_ids", post(get_started_at_by_ids))
         .route("/get_flow_debug_info/:id", get(get_flow_job_debug_info))
         .route("/completed/get/:id", get(get_completed_job))
@@ -1694,6 +1695,86 @@ async fn get_args(
     }
 }
 
+/// OTel trace span returned from the API
+#[derive(Debug, Serialize, Deserialize)]
+pub struct OtelTraceSpan {
+    pub trace_id: String,
+    pub span_id: String,
+    pub parent_span_id: Option<String>,
+    pub operation_name: String,
+    pub service_name: Option<String>,
+    pub start_time_unix_nano: i64,
+    pub end_time_unix_nano: i64,
+    pub duration_ns: i64,
+    pub status_code: i16,
+    pub status_message: Option<String>,
+    pub attributes: serde_json::Value,
+    pub events: serde_json::Value,
+}
+
+async fn get_job_otel_traces(
+    OptAuthed(opt_authed): OptAuthed,
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> JsonResult<Vec<OtelTraceSpan>> {
+    // Check if user has access to view job (similar to get_args)
+    // Use raw SQL query since this is a simple check
+    let job_record: Option<(String,)> = sqlx::query_as(
+        "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2",
+    )
+    .bind(id)
+    .bind(&w_id)
+    .fetch_optional(&db)
+    .await?;
+
+    if let Some(record) = job_record {
+        if opt_authed.is_none() && record.0 != "anonymous" {
+            return Err(Error::BadRequest(
+                "As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
+            ));
+        }
+    } else {
+        return Err(Error::NotFound(format!("Job {} not found", id)));
+    }
+
+    // Fetch OTel traces for this job using raw SQL (table is new, not in sqlx cache)
+    let traces: Vec<(String, String, Option<String>, String, Option<String>, i64, i64, Option<i64>, Option<i16>, Option<String>, Option<serde_json::Value>, Option<serde_json::Value>)> = sqlx::query_as(
+        r#"
+        SELECT
+            trace_id, span_id, parent_span_id, operation_name, service_name,
+            start_time_unix_nano, end_time_unix_nano, duration_ns,
+            status_code, status_message, attributes, events
+        FROM job_otel_traces
+        WHERE job_id = $1 AND workspace_id = $2
+        ORDER BY start_time_unix_nano ASC
+        "#,
+    )
+    .bind(id)
+    .bind(&w_id)
+    .fetch_all(&db)
+    .await?;
+
+    let spans: Vec<OtelTraceSpan> = traces
+        .into_iter()
+        .map(|row| OtelTraceSpan {
+            trace_id: row.0,
+            span_id: row.1,
+            parent_span_id: row.2,
+            operation_name: row.3,
+            service_name: row.4,
+            start_time_unix_nano: row.5,
+            end_time_unix_nano: row.6,
+            duration_ns: row.7.unwrap_or(0),
+            status_code: row.8.unwrap_or(0) as i16,
+            status_message: row.9,
+            attributes: row.10.unwrap_or(serde_json::json!({})),
+            events: row.11.unwrap_or(serde_json::json!([])),
+        })
+        .collect();
+
+    Ok(Json(spans))
+}
+
 async fn get_started_at_by_ids(
     Extension(db): Extension<DB>,
     Json(mut ids): Json<Vec<Uuid>>,

backend/windmill-common/src/global_settings.rs

@@ -46,6 +46,7 @@ pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
 pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
 pub const OTEL_SETTING: &str = "otel";
+pub const OTEL_AUTO_INSTRUMENTATION_SETTING: &str = "otel_auto_instrumentation";
 pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
 
 pub const ENV_SETTINGS: &[&str] = &[

backend/windmill-worker/Cargo.toml

@@ -12,7 +12,7 @@ path = "src/lib.rs"
 default = []
 private = []
 prometheus = ["dep:prometheus", "windmill-common/prometheus"]
-enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"]
+enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:prost", "dep:axum"]
 mssql = ["dep:tiberius"]
 bigquery = ["dep:gcp_auth"]
 benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"]
@@ -138,6 +138,8 @@ libloading = { workspace = true, optional = true }
 opentelemetry = { workspace = true, optional = true }
 bollard = { workspace = true, optional = true }
 oracle = { workspace = true, optional = true }
+prost = { workspace = true, optional = true }
+axum = { workspace = true, optional = true }
 
 [build-dependencies]
 deno_fetch = { workspace = true, optional = true }

backend/windmill-worker/otel_bun_loader.js

@@ -0,0 +1,191 @@
+// OpenTelemetry Auto-Instrumentation Loader for Bun/Node.js
+// This file is loaded via -r flag when WINDMILL_OTEL_AUTO_INSTRUMENTATION=true
+// It wraps the global fetch to automatically create OTel spans
+
+const OTEL_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
+const JOB_ID = process.env.WINDMILL_JOB_ID;
+const WORKSPACE_ID = process.env.WINDMILL_WORKSPACE_ID;
+const SERVICE_NAME = process.env.OTEL_SERVICE_NAME || 'windmill-script';
+
+if (!OTEL_ENDPOINT || !JOB_ID || !WORKSPACE_ID) {
+  // Skip instrumentation if env vars not set
+  console.log('[OTel] Missing environment variables, skipping instrumentation');
+} else {
+  console.log(`[OTel] Auto-instrumentation enabled, sending traces to ${OTEL_ENDPOINT}`);
+
+  // Simple trace context generator
+  function generateTraceId() {
+    const bytes = new Uint8Array(16);
+    crypto.getRandomValues(bytes);
+    return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
+  }
+
+  function generateSpanId() {
+    const bytes = new Uint8Array(8);
+    crypto.getRandomValues(bytes);
+    return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
+  }
+
+  // Store for pending spans
+  const pendingSpans = [];
+  let flushTimeout = null;
+
+  // Encode spans to OTLP protobuf format (simplified JSON export for now)
+  async function flushSpans() {
+    if (pendingSpans.length === 0) return;
+
+    const spansToSend = pendingSpans.splice(0, pendingSpans.length);
+
+    // Build OTLP JSON format (will be converted to protobuf by collector or use JSON endpoint)
+    const jsonEndpoint = OTEL_ENDPOINT.replace('/v1/traces', '/v1/traces');
+
+    const exportRequest = {
+      resourceSpans: [{
+        resource: {
+          attributes: [
+            { key: 'service.name', value: { stringValue: SERVICE_NAME } },
+            { key: 'windmill.job_id', value: { stringValue: JOB_ID } },
+            { key: 'windmill.workspace_id', value: { stringValue: WORKSPACE_ID } }
+          ]
+        },
+        scopeSpans: [{
+          scope: {
+            name: 'windmill-otel-bun-loader',
+            version: '1.0.0'
+          },
+          spans: spansToSend
+        }]
+      }]
+    };
+
+    try {
+      // Use the original fetch to avoid recursion
+      await originalFetch(jsonEndpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(exportRequest)
+      });
+    } catch (e) {
+      // Silently ignore export errors to not affect the script
+      console.error('[OTel] Failed to export spans:', e.message);
+    }
+  }
+
+  function scheduleFlush() {
+    if (flushTimeout) return;
+    flushTimeout = setTimeout(() => {
+      flushTimeout = null;
+      flushSpans();
+    }, 100);
+  }
+
+  // Wrap the global fetch
+  const originalFetch = globalThis.fetch;
+
+  globalThis.fetch = async function instrumentedFetch(input, init) {
+    const url = typeof input === 'string' ? input : input.url;
+    const method = init?.method || (typeof input === 'object' ? input.method : 'GET') || 'GET';
+
+    // Don't instrument calls to the OTel endpoint to avoid recursion
+    if (url.includes(OTEL_ENDPOINT) || url.includes('/v1/traces')) {
+      return originalFetch(input, init);
+    }
+
+    const traceId = generateTraceId();
+    const spanId = generateSpanId();
+    const startTimeNano = BigInt(Date.now()) * 1000000n;
+
+    let statusCode = 0; // UNSET
+    let statusMessage = '';
+    let responseStatus = 0;
+    let error = null;
+
+    try {
+      const response = await originalFetch(input, init);
+      responseStatus = response.status;
+
+      if (response.ok) {
+        statusCode = 1; // OK
+      } else {
+        statusCode = 2; // ERROR
+        statusMessage = `HTTP ${response.status}`;
+      }
+
+      return response;
+    } catch (e) {
+      statusCode = 2; // ERROR
+      statusMessage = e.message;
+      error = e;
+      throw e;
+    } finally {
+      const endTimeNano = BigInt(Date.now()) * 1000000n;
+
+      // Parse URL for attributes
+      let parsedUrl;
+      try {
+        parsedUrl = new URL(url);
+      } catch {
+        parsedUrl = { hostname: '', pathname: url, protocol: '' };
+      }
+
+      const span = {
+        traceId: hexToBase64(traceId),
+        spanId: hexToBase64(spanId),
+        name: `HTTP ${method}`,
+        kind: 3, // CLIENT
+        startTimeUnixNano: startTimeNano.toString(),
+        endTimeUnixNano: endTimeNano.toString(),
+        attributes: [
+          { key: 'http.method', value: { stringValue: method } },
+          { key: 'http.url', value: { stringValue: url } },
+          { key: 'http.host', value: { stringValue: parsedUrl.hostname } },
+          { key: 'http.target', value: { stringValue: parsedUrl.pathname } },
+          { key: 'http.scheme', value: { stringValue: parsedUrl.protocol?.replace(':', '') || 'https' } }
+        ],
+        status: {
+          code: statusCode,
+          message: statusMessage
+        }
+      };
+
+      if (responseStatus) {
+        span.attributes.push({ key: 'http.status_code', value: { intValue: responseStatus.toString() } });
+      }
+
+      if (error) {
+        span.events = [{
+          timeUnixNano: endTimeNano.toString(),
+          name: 'exception',
+          attributes: [
+            { key: 'exception.message', value: { stringValue: error.message } },
+            { key: 'exception.type', value: { stringValue: error.name } }
+          ]
+        }];
+      }
+
+      pendingSpans.push(span);
+      scheduleFlush();
+    }
+  };
+
+  // Helper to convert hex to base64 for OTLP JSON format
+  function hexToBase64(hex) {
+    const bytes = new Uint8Array(hex.match(/.{1,2}/g).map(byte => parseInt(byte, 16)));
+    return btoa(String.fromCharCode(...bytes));
+  }
+
+  // Ensure spans are flushed before process exits
+  process.on('beforeExit', async () => {
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
+    await flushSpans();
+  });
+
+  process.on('exit', () => {
+    // Synchronous - can't do async here, but beforeExit should have handled it
+  });
+}

backend/windmill-worker/src/ansible_executor.rs

@@ -13,12 +13,13 @@ use tokio::process::Command;
 use uuid::Uuid;
 use windmill_common::{
     error,
-    git_sync_oss::prepend_token_to_github_url,
     worker::{
         is_allowed_file_location, split_python_requirements, to_raw_value, write_file,
         write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG,
     },
 };
+#[cfg(feature = "enterprise")]
+use windmill_common::git_sync_oss::prepend_token_to_github_url;
 use windmill_queue::MiniPulledJob;
 
 use windmill_parser_yaml::{
@@ -948,8 +949,12 @@ pub async fn handle_ansible_job(
                 ));
             };
 
+            #[cfg(feature = "enterprise")]
             let mut secret_url = git_repo_resource.get("url").and_then(|s| s.as_str()).map(|s| s.to_string())
                 .ok_or(anyhow!("Failed to get url from git repo resource, please check that the resource has the correct type (git_repository)"))?;
+            #[cfg(not(feature = "enterprise"))]
+            let secret_url = git_repo_resource.get("url").and_then(|s| s.as_str()).map(|s| s.to_string())
+                .ok_or(anyhow!("Failed to get url from git repo resource, please check that the resource has the correct type (git_repository)"))?;
 
             #[cfg(feature = "enterprise")]
             let is_github_app = git_repo_resource.get("is_github_app").and_then(|s| s.as_bool())

backend/windmill-worker/src/bun_executor.rs

@@ -51,10 +51,21 @@ use windmill_common::s3_helpers::attempt_fetch_bytes;
 
 use windmill_parser::Typ;
 
+#[cfg(feature = "enterprise")]
+use crate::otel_auto_instrumentation_impl::{
+    get_otel_auto_instrumentation_config, get_otel_typescript_env_vars,
+};
+#[cfg(not(feature = "enterprise"))]
+use crate::otel_auto_instrumentation_oss::{
+    get_otel_auto_instrumentation_config, get_otel_typescript_env_vars,
+};
+
 const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js");
 
 const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");
 
+const OTEL_BUN_LOADER: &str = include_str!("../otel_bun_loader.js");
+
 const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
 
 pub const BUN_LOCK_SPLIT: &str = "\n//bun.lock\n";
@@ -1356,6 +1367,24 @@ try {{
     }
     append_logs(&job.id, &job.workspace_id, init_logs, conn).await;
 
+    // Get OTel auto-instrumentation env vars (EE feature)
+    let otel_envs: Vec<(String, String)> = if let Connection::Sql(db) = conn {
+        let otel_config = get_otel_auto_instrumentation_config(db).await;
+        if otel_config.enabled && otel_config.typescript_enabled {
+            get_otel_typescript_env_vars(&job.id, &job.workspace_id, job.runnable_path(), &otel_config)
+        } else {
+            vec![]
+        }
+    } else {
+        vec![]
+    };
+
+    // Write OTel loader file if auto-instrumentation is enabled
+    let otel_enabled = !otel_envs.is_empty();
+    if otel_enabled {
+        write_file(job_dir, "otel_bun_loader.js", OTEL_BUN_LOADER)?;
+    }
+
     //do not cache local dependencies
     let child = if !*DISABLE_NSJAIL {
         let _ = write_file(
@@ -1388,17 +1417,22 @@ try {{
                 "/tmp/nodejs/wrapper.mjs",
             ]
         } else if codebase.is_some() || has_bundle_cache {
-            vec![
+            let mut base_args = vec![
                 "--config",
                 "run.config.proto",
                 "--",
                 &BUN_PATH,
                 "run",
                 "--preserve-symlinks",
-                "/tmp/bun/wrapper.mjs",
-            ]
+            ];
+            // Add OTel loader if enabled
+            if otel_enabled {
+                base_args.extend_from_slice(&["-r", "/tmp/bun/otel_bun_loader.js"]);
+            }
+            base_args.push("/tmp/bun/wrapper.mjs");
+            base_args
         } else {
-            vec![
+            let mut base_args = vec![
                 "--config",
                 "run.config.proto",
                 "--",
@@ -1408,8 +1442,13 @@ try {{
                 "--prefer-offline",
                 "-r",
                 "/tmp/bun/loader.bun.js",
-                "/tmp/bun/wrapper.mjs",
-            ]
+            ];
+            // Add OTel loader if enabled
+            if otel_enabled {
+                base_args.extend_from_slice(&["-r", "/tmp/bun/otel_bun_loader.js"]);
+            }
+            base_args.push("/tmp/bun/wrapper.mjs");
+            base_args
         };
         nsjail_cmd
             .current_dir(job_dir)
@@ -1417,6 +1456,7 @@ try {{
             .envs(envs)
             .envs(reserved_variables)
             .envs(common_bun_proc_envs)
+            .envs(otel_envs.clone())
             .env("PATH", PATH_ENV.as_str())
             .args(args)
             .stdout(Stdio::piped())
@@ -1434,6 +1474,7 @@ try {{
                 .envs(envs)
                 .envs(reserved_variables)
                 .envs(common_bun_proc_envs)
+                .envs(otel_envs.clone())
                 .stdin(Stdio::null())
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped());
@@ -1444,26 +1485,40 @@ try {{
             bun_cmd
         } else {
             let script_path = format!("{job_dir}/wrapper.mjs");
+            let otel_loader_path = format!("{job_dir}/otel_bun_loader.js");
 
-            let args: Vec<&str> = if codebase.is_some() || has_bundle_cache {
-                vec!["run", &script_path]
+            let args: Vec<String> = if codebase.is_some() || has_bundle_cache {
+                let mut base_args = vec!["run".to_string()];
+                // Add OTel loader if enabled
+                if otel_enabled {
+                    base_args.extend_from_slice(&["-r".to_string(), otel_loader_path.clone()]);
+                }
+                base_args.push(script_path);
+                base_args
             } else {
-                vec![
-                    "run",
-                    "-i",
-                    "--prefer-offline",
-                    "-r",
-                    "./loader.bun.js",
-                    &script_path,
-                ]
+                let mut base_args = vec![
+                    "run".to_string(),
+                    "-i".to_string(),
+                    "--prefer-offline".to_string(),
+                    "-r".to_string(),
+                    "./loader.bun.js".to_string(),
+                ];
+                // Add OTel loader if enabled
+                if otel_enabled {
+                    base_args.extend_from_slice(&["-r".to_string(), otel_loader_path.clone()]);
+                }
+                base_args.push(script_path);
+                base_args
             };
-            let mut bun_cmd = build_command_with_isolation(&*BUN_PATH, &args);
+            let args_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect();
+            let mut bun_cmd = build_command_with_isolation(&*BUN_PATH, &args_refs);
             bun_cmd
                 .current_dir(job_dir)
                 .env_clear()
                 .envs(envs)
                 .envs(reserved_variables)
                 .envs(common_bun_proc_envs)
+                .envs(otel_envs)
                 .stdin(Stdio::null())
                 .stdout(Stdio::piped())
                 .stderr(Stdio::piped());

backend/windmill-worker/src/deno_executor.rs

@@ -23,6 +23,15 @@ use windmill_common::{
 };
 use windmill_parser::Typ;
 
+#[cfg(feature = "enterprise")]
+use crate::otel_auto_instrumentation_impl::{
+    get_otel_auto_instrumentation_config, get_otel_typescript_env_vars,
+};
+#[cfg(not(feature = "enterprise"))]
+use crate::otel_auto_instrumentation_oss::{
+    get_otel_auto_instrumentation_config, get_otel_typescript_env_vars,
+};
+
 lazy_static::lazy_static! {
 
     static ref DENO_FLAGS: Option<Vec<String>> = std::env::var("DENO_FLAGS")
@@ -357,6 +366,18 @@ try {{
         common_deno_proc_envs.insert("HOME".to_string(), job_dir.to_string());
     }
 
+    // Get OTel auto-instrumentation env vars (EE feature)
+    let otel_envs: Vec<(String, String)> = if let Connection::Sql(db) = conn {
+        let otel_config = get_otel_auto_instrumentation_config(db).await;
+        if otel_config.enabled && otel_config.typescript_enabled {
+            get_otel_typescript_env_vars(&job.id, &job.workspace_id, job.runnable_path(), &otel_config)
+        } else {
+            vec![]
+        }
+    } else {
+        vec![]
+    };
+
     //do not cache local dependencies
     let child = {
         let reload = format!("--reload={base_internal_url}");
@@ -417,6 +438,7 @@ try {{
             .envs(envs)
             .envs(reserved_variables)
             .envs(common_deno_proc_envs)
+            .envs(otel_envs)
             .stdin(Stdio::null())
             .stdout(Stdio::piped())
             .stderr(Stdio::piped());

backend/windmill-worker/src/lib.rs

@@ -52,6 +52,9 @@ mod oracledb_executor;
 #[cfg(feature = "private")]
 pub mod otel_ee;
 mod otel_oss;
+#[cfg(feature = "enterprise")]
+pub mod otel_auto_instrumentation_impl;
+pub mod otel_auto_instrumentation_oss;
 mod pg_executor;
 #[cfg(feature = "php")]
 mod php_executor;

backend/windmill-worker/src/otel_auto_instrumentation_impl.rs

@@ -0,0 +1,777 @@
+//! OTel Auto-Instrumentation Collector - Enterprise Edition
+//!
+//! This module provides the full implementation for OTel auto-instrumentation
+//! collector for Windmill Enterprise Edition.
+//!
+//! Features:
+//! - Config caching from global_settings with TTL
+//! - Environment variable generation for Python and TypeScript
+//! - OTLP trace parsing (JSON format)
+//! - Built-in HTTP collector server
+//! - Database storage for spans
+
+use serde::{Deserialize, Serialize};
+use sqlx::{Pool, Postgres};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::sync::RwLock;
+use uuid::Uuid;
+
+use windmill_common::global_settings::{
+    load_value_from_global_settings, OTEL_AUTO_INSTRUMENTATION_SETTING,
+};
+
+/// Configuration for OTel auto-instrumentation
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct OtelAutoInstrumentationConfig {
+    pub enabled: bool,
+    pub python_enabled: bool,
+    pub typescript_enabled: bool,
+    pub collector_port: u16,
+}
+
+impl OtelAutoInstrumentationConfig {
+    pub fn default_config() -> Self {
+        Self {
+            enabled: false,
+            python_enabled: true,
+            typescript_enabled: true,
+            collector_port: 4318,
+        }
+    }
+}
+
+/// OTel span received from auto-instrumented scripts
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OtelSpan {
+    pub trace_id: String,
+    pub span_id: String,
+    pub parent_span_id: Option<String>,
+    pub operation_name: String,
+    pub service_name: Option<String>,
+    pub start_time_unix_nano: i64,
+    pub end_time_unix_nano: i64,
+    pub status_code: i16,
+    pub status_message: Option<String>,
+    pub attributes: serde_json::Value,
+    pub events: serde_json::Value,
+}
+
+/// Cached configuration with TTL
+struct CachedConfig {
+    config: OtelAutoInstrumentationConfig,
+    fetched_at: Instant,
+}
+
+// Global config cache with 60-second TTL
+lazy_static::lazy_static! {
+    static ref CONFIG_CACHE: Arc<RwLock<Option<CachedConfig>>> = Arc::new(RwLock::new(None));
+}
+
+const CONFIG_CACHE_TTL: Duration = Duration::from_secs(60);
+
+/// Check if OTel auto-instrumentation is enabled
+pub async fn is_otel_auto_instrumentation_enabled(db: &Pool<Postgres>) -> bool {
+    let config = get_otel_auto_instrumentation_config(db).await;
+    config.enabled
+}
+
+/// Get OTel auto-instrumentation config with caching
+pub async fn get_otel_auto_instrumentation_config(
+    db: &Pool<Postgres>,
+) -> OtelAutoInstrumentationConfig {
+    // Try to read from cache first
+    {
+        let cache = CONFIG_CACHE.read().await;
+        if let Some(ref cached) = *cache {
+            if cached.fetched_at.elapsed() < CONFIG_CACHE_TTL {
+                return cached.config.clone();
+            }
+        }
+    }
+
+    // Cache miss or expired, fetch from database
+    let config = match load_value_from_global_settings(db, OTEL_AUTO_INSTRUMENTATION_SETTING).await
+    {
+        Ok(Some(value)) => {
+            serde_json::from_value(value).unwrap_or_else(|_| OtelAutoInstrumentationConfig::default_config())
+        }
+        _ => OtelAutoInstrumentationConfig::default_config(),
+    };
+
+    // Update cache
+    {
+        let mut cache = CONFIG_CACHE.write().await;
+        *cache = Some(CachedConfig {
+            config: config.clone(),
+            fetched_at: Instant::now(),
+        });
+    }
+
+    config
+}
+
+/// Get OTel environment variables for Python scripts
+pub fn get_otel_python_env_vars(
+    job_id: &Uuid,
+    workspace_id: &str,
+    script_path: &str,
+    config: &OtelAutoInstrumentationConfig,
+) -> Vec<(String, String)> {
+    if !config.enabled || !config.python_enabled {
+        return vec![];
+    }
+
+    let service_name = format!("windmill-python-{}", script_path.replace('/', "-"));
+    let endpoint = format!("http://127.0.0.1:{}/v1/traces", config.collector_port);
+
+    vec![
+        ("WINDMILL_OTEL_AUTO_INSTRUMENTATION".to_string(), "true".to_string()),
+        ("WINDMILL_JOB_ID".to_string(), job_id.to_string()),
+        ("WINDMILL_WORKSPACE_ID".to_string(), workspace_id.to_string()),
+        ("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT".to_string(), endpoint),
+        ("OTEL_SERVICE_NAME".to_string(), service_name),
+        (
+            "OTEL_RESOURCE_ATTRIBUTES".to_string(),
+            format!(
+                "service.instance.id={},windmill.job_id={},windmill.workspace_id={},windmill.script_path={}",
+                job_id, job_id, workspace_id, script_path
+            ),
+        ),
+        // Python-specific OTel instrumentation settings
+        ("OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED".to_string(), "false".to_string()),
+        ("OTEL_TRACES_EXPORTER".to_string(), "otlp".to_string()),
+        ("OTEL_EXPORTER_OTLP_PROTOCOL".to_string(), "http/json".to_string()),
+    ]
+}
+
+/// Get OTel environment variables for TypeScript scripts (Bun/Deno)
+pub fn get_otel_typescript_env_vars(
+    job_id: &Uuid,
+    workspace_id: &str,
+    script_path: &str,
+    config: &OtelAutoInstrumentationConfig,
+) -> Vec<(String, String)> {
+    if !config.enabled || !config.typescript_enabled {
+        return vec![];
+    }
+
+    let service_name = format!("windmill-typescript-{}", script_path.replace('/', "-"));
+    let endpoint = format!("http://127.0.0.1:{}/v1/traces", config.collector_port);
+
+    vec![
+        ("WINDMILL_OTEL_AUTO_INSTRUMENTATION".to_string(), "true".to_string()),
+        ("WINDMILL_JOB_ID".to_string(), job_id.to_string()),
+        ("WINDMILL_WORKSPACE_ID".to_string(), workspace_id.to_string()),
+        ("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT".to_string(), endpoint),
+        ("OTEL_SERVICE_NAME".to_string(), service_name),
+        (
+            "OTEL_RESOURCE_ATTRIBUTES".to_string(),
+            format!(
+                "service.instance.id={},windmill.job_id={},windmill.workspace_id={},windmill.script_path={}",
+                job_id, job_id, workspace_id, script_path
+            ),
+        ),
+        ("OTEL_TRACES_EXPORTER".to_string(), "otlp".to_string()),
+        ("OTEL_EXPORTER_OTLP_PROTOCOL".to_string(), "http/json".to_string()),
+    ]
+}
+
+/// Store OTel spans in the database
+pub async fn store_otel_spans(
+    db: &Pool<Postgres>,
+    job_id: &Uuid,
+    workspace_id: &str,
+    spans: Vec<OtelSpan>,
+) -> anyhow::Result<()> {
+    if spans.is_empty() {
+        return Ok(());
+    }
+
+    for span in spans {
+        sqlx::query(
+            r#"
+            INSERT INTO job_otel_traces (
+                job_id, workspace_id, trace_id, span_id, parent_span_id,
+                operation_name, service_name, start_time_unix_nano, end_time_unix_nano,
+                status_code, status_message, attributes, events
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+            "#,
+        )
+        .bind(job_id)
+        .bind(workspace_id)
+        .bind(&span.trace_id)
+        .bind(&span.span_id)
+        .bind(&span.parent_span_id)
+        .bind(&span.operation_name)
+        .bind(&span.service_name)
+        .bind(span.start_time_unix_nano)
+        .bind(span.end_time_unix_nano)
+        .bind(span.status_code)
+        .bind(&span.status_message)
+        .bind(&span.attributes)
+        .bind(&span.events)
+        .execute(db)
+        .await?;
+    }
+
+    Ok(())
+}
+
+// ============================================================================
+// OTLP JSON Parsing
+// ============================================================================
+
+/// OTLP ExportTraceServiceRequest (JSON format)
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpExportTraceServiceRequest {
+    resource_spans: Option<Vec<OtlpResourceSpans>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpResourceSpans {
+    resource: Option<OtlpResource>,
+    scope_spans: Option<Vec<OtlpScopeSpans>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OtlpResource {
+    attributes: Option<Vec<OtlpKeyValue>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpScopeSpans {
+    #[allow(dead_code)]
+    scope: Option<OtlpInstrumentationScope>,
+    spans: Option<Vec<OtlpSpan>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OtlpInstrumentationScope {
+    #[allow(dead_code)]
+    name: Option<String>,
+    #[allow(dead_code)]
+    version: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpSpan {
+    trace_id: Option<String>,
+    span_id: Option<String>,
+    parent_span_id: Option<String>,
+    name: Option<String>,
+    #[allow(dead_code)]
+    kind: Option<i32>,
+    start_time_unix_nano: Option<String>,
+    end_time_unix_nano: Option<String>,
+    attributes: Option<Vec<OtlpKeyValue>>,
+    events: Option<Vec<OtlpEvent>>,
+    status: Option<OtlpStatus>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OtlpKeyValue {
+    key: String,
+    value: Option<OtlpAnyValue>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpAnyValue {
+    string_value: Option<String>,
+    int_value: Option<String>,
+    double_value: Option<f64>,
+    bool_value: Option<bool>,
+    #[allow(dead_code)]
+    array_value: Option<serde_json::Value>,
+    #[allow(dead_code)]
+    kvlist_value: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct OtlpEvent {
+    #[allow(dead_code)]
+    time_unix_nano: Option<String>,
+    name: Option<String>,
+    attributes: Option<Vec<OtlpKeyValue>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OtlpStatus {
+    code: Option<i32>,
+    message: Option<String>,
+}
+
+/// Convert OTLP attributes to JSON value
+fn attributes_to_json(attrs: &Option<Vec<OtlpKeyValue>>) -> serde_json::Value {
+    match attrs {
+        None => serde_json::json!({}),
+        Some(attrs) => {
+            let mut map = serde_json::Map::new();
+            for attr in attrs {
+                if let Some(ref value) = attr.value {
+                    let json_value = if let Some(ref s) = value.string_value {
+                        serde_json::Value::String(s.clone())
+                    } else if let Some(ref i) = value.int_value {
+                        // int_value comes as string in JSON format
+                        match i.parse::<i64>() {
+                            Ok(n) => serde_json::Value::Number(n.into()),
+                            Err(_) => serde_json::Value::String(i.clone()),
+                        }
+                    } else if let Some(d) = value.double_value {
+                        serde_json::Number::from_f64(d)
+                            .map(serde_json::Value::Number)
+                            .unwrap_or(serde_json::Value::Null)
+                    } else if let Some(b) = value.bool_value {
+                        serde_json::Value::Bool(b)
+                    } else {
+                        serde_json::Value::Null
+                    };
+                    map.insert(attr.key.clone(), json_value);
+                }
+            }
+            serde_json::Value::Object(map)
+        }
+    }
+}
+
+/// Convert OTLP events to JSON value
+fn events_to_json(events: &Option<Vec<OtlpEvent>>) -> serde_json::Value {
+    match events {
+        None => serde_json::json!([]),
+        Some(events) => {
+            let arr: Vec<serde_json::Value> = events
+                .iter()
+                .map(|e| {
+                    serde_json::json!({
+                        "name": e.name,
+                        "attributes": attributes_to_json(&e.attributes)
+                    })
+                })
+                .collect();
+            serde_json::Value::Array(arr)
+        }
+    }
+}
+
+/// Convert base64-encoded bytes to hex string
+fn base64_to_hex(b64: &str) -> String {
+    use base64::Engine;
+    match base64::engine::general_purpose::STANDARD.decode(b64) {
+        Ok(bytes) => bytes_to_hex(&bytes),
+        Err(_) => b64.to_string(), // Return as-is if not valid base64
+    }
+}
+
+/// Convert bytes to hex string
+fn bytes_to_hex(bytes: &[u8]) -> String {
+    bytes.iter().map(|b| format!("{:02x}", b)).collect()
+}
+
+/// Parse OTLP JSON request and extract spans
+pub fn parse_otlp_traces_json(
+    body: &[u8],
+    job_id: &Uuid,
+    workspace_id: &str,
+) -> anyhow::Result<Vec<OtelSpan>> {
+    let request: OtlpExportTraceServiceRequest = serde_json::from_slice(body)?;
+
+    let mut spans = Vec::new();
+
+    if let Some(resource_spans) = request.resource_spans {
+        for rs in resource_spans {
+            // Extract service name from resource attributes
+            let service_name = rs
+                .resource
+                .as_ref()
+                .and_then(|r| r.attributes.as_ref())
+                .and_then(|attrs| {
+                    attrs.iter().find(|kv| kv.key == "service.name").and_then(|kv| {
+                        kv.value.as_ref().and_then(|v| v.string_value.clone())
+                    })
+                });
+
+            if let Some(scope_spans) = rs.scope_spans {
+                for ss in scope_spans {
+                    if let Some(otlp_spans) = ss.spans {
+                        for span in otlp_spans {
+                            // Parse trace_id and span_id (base64 encoded in JSON format)
+                            let trace_id = span
+                                .trace_id
+                                .map(|id| base64_to_hex(&id))
+                                .unwrap_or_default();
+                            let span_id = span
+                                .span_id
+                                .map(|id| base64_to_hex(&id))
+                                .unwrap_or_default();
+                            let parent_span_id = span.parent_span_id.map(|id| base64_to_hex(&id));
+
+                            let start_time: i64 = span
+                                .start_time_unix_nano
+                                .as_ref()
+                                .and_then(|s| s.parse().ok())
+                                .unwrap_or(0);
+                            let end_time: i64 = span
+                                .end_time_unix_nano
+                                .as_ref()
+                                .and_then(|s| s.parse().ok())
+                                .unwrap_or(0);
+
+                            let status_code = span
+                                .status
+                                .as_ref()
+                                .and_then(|s| s.code)
+                                .unwrap_or(0) as i16;
+                            let status_message = span.status.as_ref().and_then(|s| s.message.clone());
+
+                            spans.push(OtelSpan {
+                                trace_id,
+                                span_id,
+                                parent_span_id,
+                                operation_name: span.name.unwrap_or_else(|| "unknown".to_string()),
+                                service_name: service_name.clone(),
+                                start_time_unix_nano: start_time,
+                                end_time_unix_nano: end_time,
+                                status_code,
+                                status_message,
+                                attributes: attributes_to_json(&span.attributes),
+                                events: events_to_json(&span.events),
+                            });
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Override with the actual job context
+    for span in &mut spans {
+        // Add job metadata to attributes if not present
+        if let serde_json::Value::Object(ref mut map) = span.attributes {
+            if !map.contains_key("windmill.job_id") {
+                map.insert(
+                    "windmill.job_id".to_string(),
+                    serde_json::Value::String(job_id.to_string()),
+                );
+            }
+            if !map.contains_key("windmill.workspace_id") {
+                map.insert(
+                    "windmill.workspace_id".to_string(),
+                    serde_json::Value::String(workspace_id.to_string()),
+                );
+            }
+        }
+    }
+
+    Ok(spans)
+}
+
+// ============================================================================
+// Built-in HTTP Collector Server
+// ============================================================================
+
+use axum::{
+    body::Bytes,
+    extract::{Path, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::post,
+    Router,
+};
+
+/// Application state for the collector server
+#[derive(Clone)]
+struct CollectorState {
+    db: Pool<Postgres>,
+}
+
+/// Handle OTLP traces endpoint
+async fn handle_traces(
+    State(state): State<CollectorState>,
+    Path((workspace_id, job_id)): Path<(String, String)>,
+    body: Bytes,
+) -> impl IntoResponse {
+    let job_uuid = match Uuid::parse_str(&job_id) {
+        Ok(id) => id,
+        Err(_) => {
+            return (StatusCode::BAD_REQUEST, "Invalid job ID");
+        }
+    };
+
+    match parse_otlp_traces_json(&body, &job_uuid, &workspace_id) {
+        Ok(spans) => {
+            if let Err(e) = store_otel_spans(&state.db, &job_uuid, &workspace_id, spans).await {
+                tracing::error!("Failed to store OTel spans: {}", e);
+                return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to store spans");
+            }
+            (StatusCode::OK, "")
+        }
+        Err(e) => {
+            tracing::error!("Failed to parse OTLP traces: {}", e);
+            (StatusCode::BAD_REQUEST, "Failed to parse traces")
+        }
+    }
+}
+
+/// Handle generic OTLP traces endpoint (extracts job info from body)
+async fn handle_traces_generic(State(state): State<CollectorState>, body: Bytes) -> impl IntoResponse {
+    // Try to extract job_id and workspace_id from the request body's resource attributes
+    let request: Result<OtlpExportTraceServiceRequest, _> = serde_json::from_slice(&body);
+
+    let (job_id, workspace_id) = match &request {
+        Ok(req) => {
+            let mut job_id = None;
+            let mut workspace_id = None;
+
+            if let Some(ref resource_spans) = req.resource_spans {
+                for rs in resource_spans {
+                    if let Some(ref resource) = rs.resource {
+                        if let Some(ref attrs) = resource.attributes {
+                            for attr in attrs {
+                                if attr.key == "windmill.job_id" {
+                                    job_id = attr
+                                        .value
+                                        .as_ref()
+                                        .and_then(|v| v.string_value.clone());
+                                } else if attr.key == "windmill.workspace_id" {
+                                    workspace_id = attr
+                                        .value
+                                        .as_ref()
+                                        .and_then(|v| v.string_value.clone());
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            (job_id, workspace_id)
+        }
+        Err(_) => (None, None),
+    };
+
+    match (job_id, workspace_id) {
+        (Some(job_id_str), Some(ws_id)) => {
+            let job_uuid = match Uuid::parse_str(&job_id_str) {
+                Ok(id) => id,
+                Err(_) => {
+                    return (StatusCode::BAD_REQUEST, "Invalid job ID in resource attributes");
+                }
+            };
+
+            match parse_otlp_traces_json(&body, &job_uuid, &ws_id) {
+                Ok(spans) => {
+                    if let Err(e) = store_otel_spans(&state.db, &job_uuid, &ws_id, spans).await {
+                        tracing::error!("Failed to store OTel spans: {}", e);
+                        return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to store spans");
+                    }
+                    (StatusCode::OK, "")
+                }
+                Err(e) => {
+                    tracing::error!("Failed to parse OTLP traces: {}", e);
+                    (StatusCode::BAD_REQUEST, "Failed to parse traces")
+                }
+            }
+        }
+        _ => (
+            StatusCode::BAD_REQUEST,
+            "Missing windmill.job_id or windmill.workspace_id in resource attributes",
+        ),
+    }
+}
+
+/// Start the built-in OTel collector HTTP server
+pub async fn start_otel_collector_server(
+    db: Pool<Postgres>,
+    config: OtelAutoInstrumentationConfig,
+) -> anyhow::Result<()> {
+    if !config.enabled {
+        tracing::info!("OTel auto-instrumentation is disabled, not starting collector server");
+        return Ok(());
+    }
+
+    let state = CollectorState { db };
+
+    let app = Router::new()
+        // Path-based endpoint for explicit job context
+        .route("/v1/traces/:workspace_id/:job_id", post(handle_traces))
+        // Generic endpoint that extracts job info from resource attributes
+        .route("/v1/traces", post(handle_traces_generic))
+        .with_state(state);
+
+    let addr = format!("127.0.0.1:{}", config.collector_port);
+    tracing::info!("Starting OTel collector server on {}", addr);
+
+    let listener = tokio::net::TcpListener::bind(&addr).await?;
+    axum::serve(listener, app)
+        .await
+        .map_err(|e| anyhow::anyhow!("OTel collector server error: {}", e))?;
+
+    Ok(())
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_bytes_to_hex() {
+        assert_eq!(bytes_to_hex(&[0x12, 0x34, 0xab, 0xcd]), "1234abcd");
+        assert_eq!(bytes_to_hex(&[]), "");
+    }
+
+    #[test]
+    fn test_base64_to_hex() {
+        // "EjSrzQ==" is base64 for [0x12, 0x34, 0xab, 0xcd]
+        let b64 = base64::Engine::encode(
+            &base64::engine::general_purpose::STANDARD,
+            &[0x12, 0x34, 0xab, 0xcd],
+        );
+        assert_eq!(base64_to_hex(&b64), "1234abcd");
+    }
+
+    #[test]
+    fn test_attributes_to_json() {
+        let attrs = vec![
+            OtlpKeyValue {
+                key: "string_attr".to_string(),
+                value: Some(OtlpAnyValue {
+                    string_value: Some("hello".to_string()),
+                    int_value: None,
+                    double_value: None,
+                    bool_value: None,
+                    array_value: None,
+                    kvlist_value: None,
+                }),
+            },
+            OtlpKeyValue {
+                key: "int_attr".to_string(),
+                value: Some(OtlpAnyValue {
+                    string_value: None,
+                    int_value: Some("42".to_string()),
+                    double_value: None,
+                    bool_value: None,
+                    array_value: None,
+                    kvlist_value: None,
+                }),
+            },
+        ];
+
+        let json = attributes_to_json(&Some(attrs));
+        assert_eq!(json["string_attr"], "hello");
+        assert_eq!(json["int_attr"], 42);
+    }
+
+    #[test]
+    fn test_parse_otlp_traces_json() {
+        let json_body = r#"{
+            "resourceSpans": [{
+                "resource": {
+                    "attributes": [
+                        {"key": "service.name", "value": {"stringValue": "test-service"}}
+                    ]
+                },
+                "scopeSpans": [{
+                    "scope": {"name": "test-scope"},
+                    "spans": [{
+                        "traceId": "EjSrzQ==",
+                        "spanId": "VGWN0g==",
+                        "name": "test-span",
+                        "startTimeUnixNano": "1704067200000000000",
+                        "endTimeUnixNano": "1704067201000000000",
+                        "attributes": [
+                            {"key": "http.method", "value": {"stringValue": "GET"}}
+                        ],
+                        "status": {"code": 1}
+                    }]
+                }]
+            }]
+        }"#;
+
+        let job_id = Uuid::new_v4();
+        let workspace_id = "test-workspace";
+
+        let spans = parse_otlp_traces_json(json_body.as_bytes(), &job_id, workspace_id).unwrap();
+        assert_eq!(spans.len(), 1);
+        assert_eq!(spans[0].operation_name, "test-span");
+        assert_eq!(spans[0].service_name, Some("test-service".to_string()));
+        assert_eq!(spans[0].status_code, 1);
+    }
+
+    #[test]
+    fn test_get_otel_typescript_env_vars() {
+        let job_id = Uuid::new_v4();
+        let workspace_id = "test-ws";
+        let script_path = "u/user/test_script";
+        let config = OtelAutoInstrumentationConfig {
+            enabled: true,
+            python_enabled: true,
+            typescript_enabled: true,
+            collector_port: 4318,
+        };
+
+        let env_vars = get_otel_typescript_env_vars(&job_id, workspace_id, script_path, &config);
+        assert!(!env_vars.is_empty());
+
+        let env_map: std::collections::HashMap<_, _> = env_vars.into_iter().collect();
+        assert_eq!(env_map.get("WINDMILL_OTEL_AUTO_INSTRUMENTATION"), Some(&"true".to_string()));
+        assert_eq!(env_map.get("WINDMILL_JOB_ID"), Some(&job_id.to_string()));
+        assert!(env_map.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT").is_some());
+    }
+
+    #[test]
+    fn test_get_otel_typescript_env_vars_disabled() {
+        let job_id = Uuid::new_v4();
+        let workspace_id = "test-ws";
+        let script_path = "u/user/test_script";
+        let config = OtelAutoInstrumentationConfig {
+            enabled: true,
+            python_enabled: true,
+            typescript_enabled: false, // Disabled for TypeScript
+            collector_port: 4318,
+        };
+
+        let env_vars = get_otel_typescript_env_vars(&job_id, workspace_id, script_path, &config);
+        assert!(env_vars.is_empty());
+    }
+
+    #[test]
+    fn test_json_attributes_to_value() {
+        let attrs = Some(vec![
+            OtlpKeyValue {
+                key: "bool_val".to_string(),
+                value: Some(OtlpAnyValue {
+                    string_value: None,
+                    int_value: None,
+                    double_value: None,
+                    bool_value: Some(true),
+                    array_value: None,
+                    kvlist_value: None,
+                }),
+            },
+            OtlpKeyValue {
+                key: "double_val".to_string(),
+                value: Some(OtlpAnyValue {
+                    string_value: None,
+                    int_value: None,
+                    double_value: Some(3.14),
+                    bool_value: None,
+                    array_value: None,
+                    kvlist_value: None,
+                }),
+            },
+        ]);
+
+        let json = attributes_to_json(&attrs);
+        assert_eq!(json["bool_val"], true);
+        assert!((json["double_val"].as_f64().unwrap() - 3.14).abs() < 0.001);
+    }
+}

backend/windmill-worker/src/otel_auto_instrumentation_oss.rs

@@ -0,0 +1,108 @@
+//! OTel Auto-Instrumentation Collector - OSS Stub
+//!
+//! This module provides stub implementations for the OTel auto-instrumentation
+//! collector. The actual implementation is in the EE version.
+
+#[cfg(feature = "enterprise")]
+#[allow(unused)]
+pub use crate::otel_auto_instrumentation_impl::*;
+
+use serde::{Deserialize, Serialize};
+
+#[cfg(not(feature = "enterprise"))]
+use uuid::Uuid;
+
+/// Configuration for OTel auto-instrumentation
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct OtelAutoInstrumentationConfig {
+    pub enabled: bool,
+    pub python_enabled: bool,
+    pub typescript_enabled: bool,
+    pub collector_port: u16,
+}
+
+impl OtelAutoInstrumentationConfig {
+    pub fn default_config() -> Self {
+        Self {
+            enabled: false,
+            python_enabled: true,
+            typescript_enabled: true,
+            collector_port: 4318,
+        }
+    }
+}
+
+/// OTel span received from auto-instrumented scripts
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OtelSpan {
+    pub trace_id: String,
+    pub span_id: String,
+    pub parent_span_id: Option<String>,
+    pub operation_name: String,
+    pub service_name: Option<String>,
+    pub start_time_unix_nano: i64,
+    pub end_time_unix_nano: i64,
+    pub status_code: i16,
+    pub status_message: Option<String>,
+    pub attributes: serde_json::Value,
+    pub events: serde_json::Value,
+}
+
+/// OSS stub: Check if OTel auto-instrumentation is enabled
+#[cfg(not(feature = "enterprise"))]
+pub async fn is_otel_auto_instrumentation_enabled(
+    _db: &sqlx::Pool<sqlx::Postgres>,
+) -> bool {
+    false
+}
+
+/// OSS stub: Get OTel auto-instrumentation config
+#[cfg(not(feature = "enterprise"))]
+pub async fn get_otel_auto_instrumentation_config(
+    _db: &sqlx::Pool<sqlx::Postgres>,
+) -> OtelAutoInstrumentationConfig {
+    OtelAutoInstrumentationConfig::default_config()
+}
+
+/// OSS stub: Get OTel environment variables for Python scripts
+#[cfg(not(feature = "enterprise"))]
+pub fn get_otel_python_env_vars(
+    _job_id: &Uuid,
+    _workspace_id: &str,
+    _script_path: &str,
+    _config: &OtelAutoInstrumentationConfig,
+) -> Vec<(String, String)> {
+    vec![]
+}
+
+/// OSS stub: Get OTel environment variables for TypeScript scripts (Bun/Deno)
+#[cfg(not(feature = "enterprise"))]
+pub fn get_otel_typescript_env_vars(
+    _job_id: &Uuid,
+    _workspace_id: &str,
+    _script_path: &str,
+    _config: &OtelAutoInstrumentationConfig,
+) -> Vec<(String, String)> {
+    vec![]
+}
+
+/// OSS stub: Store OTel spans in the database
+#[cfg(not(feature = "enterprise"))]
+pub async fn store_otel_spans(
+    _db: &sqlx::Pool<sqlx::Postgres>,
+    _job_id: &Uuid,
+    _workspace_id: &str,
+    _spans: Vec<OtelSpan>,
+) -> anyhow::Result<()> {
+    Ok(())
+}
+
+/// OSS stub: Start the built-in OTel collector HTTP server
+#[cfg(not(feature = "enterprise"))]
+pub async fn start_otel_collector_server(
+    _db: sqlx::Pool<sqlx::Postgres>,
+    _config: OtelAutoInstrumentationConfig,
+) -> anyhow::Result<()> {
+    // No-op in OSS
+    Ok(())
+}

backend/windmill-worker/src/python_executor.rs

@@ -41,6 +41,15 @@ use windmill_common::variables::get_secret_value_as_admin;
 use std::env::var;
 use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo};
 
+#[cfg(feature = "enterprise")]
+use crate::otel_auto_instrumentation_impl::{
+    get_otel_auto_instrumentation_config, get_otel_python_env_vars,
+};
+#[cfg(not(feature = "enterprise"))]
+use crate::otel_auto_instrumentation_oss::{
+    get_otel_auto_instrumentation_config, get_otel_python_env_vars,
+};
+
 use process_wrap::tokio::TokioChildWrapper;
 
 lazy_static::lazy_static! {
@@ -812,6 +821,18 @@ mount {{
         job.id
     );
 
+    // Get OTel auto-instrumentation env vars (EE feature)
+    let otel_envs: Vec<(String, String)> = if let Connection::Sql(db) = conn {
+        let otel_config = get_otel_auto_instrumentation_config(db).await;
+        if otel_config.enabled && otel_config.python_enabled {
+            get_otel_python_env_vars(&job.id, &job.workspace_id, &script_path, &otel_config)
+        } else {
+            vec![]
+        }
+    } else {
+        vec![]
+    };
+
     let child = if !*DISABLE_NSJAIL {
         let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
         nsjail_cmd
@@ -820,6 +841,7 @@ mount {{
             // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
             .envs(reserved_variables)
             .envs(PROXY_ENVS.clone())
+            .envs(otel_envs.clone())
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)
@@ -845,6 +867,7 @@ mount {{
             .env_clear()
             .envs(envs)
             .envs(reserved_variables)
+            .envs(otel_envs)
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
             .env("BASE_INTERNAL_URL", base_internal_url)

frontend/src/lib/components/InstanceSetting.svelte

@@ -830,6 +830,55 @@
 						</div> -->
 						{/if}
 					</div>
+				{:else if setting.fieldType == 'otel_auto_instrumentation'}
+					<div class="flex flex-col gap-4 border rounded p-4">
+						{#if $values[setting.key]}
+							<div class="flex gap-8">
+								<Toggle
+									disabled={!$enterpriseLicense}
+									id="otel_auto_instr_enabled"
+									bind:checked={$values[setting.key].enabled}
+									options={{ right: 'Enabled' }}
+								/>
+							</div>
+							{#if $values[setting.key].enabled}
+								<div class="flex gap-8">
+									<Toggle
+										disabled={!$enterpriseLicense}
+										id="otel_auto_instr_python"
+										bind:checked={$values[setting.key].python_enabled}
+										options={{ right: 'Python' }}
+									/>
+									<Toggle
+										disabled={!$enterpriseLicense}
+										id="otel_auto_instr_typescript"
+										bind:checked={$values[setting.key].typescript_enabled}
+										options={{ right: 'TypeScript (Bun/Deno)' }}
+									/>
+								</div>
+								<div class="flex flex-col gap-1">
+									<label
+										for="otel_collector_port"
+										class="block text-xs font-semibold text-emphasis">Collector Port</label
+									>
+									<TextInput
+										inputProps={{
+											type: 'number',
+											placeholder: '4318',
+											id: 'otel_collector_port',
+											disabled: !$enterpriseLicense
+										}}
+										bind:value={$values[setting.key].collector_port}
+									/>
+									<span class="text-tertiary text-xs">Port for the built-in OTel collector (default: 4318)</span>
+								</div>
+							{/if}
+						{:else}
+							<div class="text-secondary text-sm">
+								Click Save to initialize OTel auto-instrumentation settings
+							</div>
+						{/if}
+					</div>
 				{:else if setting.fieldType == 'object_store_config'}
 					<ObjectStoreConfigSettings bind:bucket_config={$values[setting.key]} />
 					<div class="mb-6"></div>

frontend/src/lib/components/JobOtelTraces.svelte

@@ -0,0 +1,254 @@
+<script lang="ts">
+	import { workspaceStore } from '$lib/stores'
+	import { onMount } from 'svelte'
+	import { Alert, Skeleton } from './common'
+	import { Activity } from 'lucide-svelte'
+
+	interface OtelTraceSpan {
+		trace_id: string
+		span_id: string
+		parent_span_id: string | null
+		operation_name: string
+		service_name: string | null
+		start_time_unix_nano: number
+		end_time_unix_nano: number
+		duration_ns: number
+		status_code: number
+		status_message: string | null
+		attributes: Record<string, any>
+		events: any[]
+	}
+
+	interface Props {
+		jobId: string
+	}
+
+	let { jobId }: Props = $props()
+	let traces: OtelTraceSpan[] = $state([])
+	let loading = $state(true)
+	let error: string | null = $state(null)
+	let expandedSpans: Set<string> = $state(new Set())
+
+	onMount(async () => {
+		await loadTraces()
+	})
+
+	async function loadTraces() {
+		if (!$workspaceStore || !jobId) return
+
+		loading = true
+		error = null
+
+		try {
+			const response = await fetch(
+				`/api/w/${$workspaceStore}/jobs/get_otel_traces/${jobId}`
+			)
+			if (response.ok) {
+				traces = await response.json()
+			} else if (response.status === 404) {
+				traces = []
+			} else {
+				error = `Failed to load traces: ${response.statusText}`
+			}
+		} catch (e) {
+			error = `Error loading traces: ${e}`
+		} finally {
+			loading = false
+		}
+	}
+
+	function formatDuration(ns: number): string {
+		if (ns < 1000) return `${ns}ns`
+		if (ns < 1000000) return `${(ns / 1000).toFixed(2)}μs`
+		if (ns < 1000000000) return `${(ns / 1000000).toFixed(2)}ms`
+		return `${(ns / 1000000000).toFixed(2)}s`
+	}
+
+	function formatTimestamp(ns: number): string {
+		const date = new Date(ns / 1000000)
+		return date.toISOString()
+	}
+
+	function getStatusColor(statusCode: number): string {
+		switch (statusCode) {
+			case 0: // Unset
+				return 'text-secondary'
+			case 1: // Ok
+				return 'text-green-600'
+			case 2: // Error
+				return 'text-red-600'
+			default:
+				return 'text-secondary'
+		}
+	}
+
+	function getStatusLabel(statusCode: number): string {
+		switch (statusCode) {
+			case 0:
+				return 'Unset'
+			case 1:
+				return 'OK'
+			case 2:
+				return 'Error'
+			default:
+				return 'Unknown'
+		}
+	}
+
+	function toggleSpan(spanId: string) {
+		const newSet = new Set(expandedSpans)
+		if (newSet.has(spanId)) {
+			newSet.delete(spanId)
+		} else {
+			newSet.add(spanId)
+		}
+		expandedSpans = newSet
+	}
+
+	// Calculate timeline metrics
+	function getTimelineMetrics(spans: OtelTraceSpan[]) {
+		if (spans.length === 0) return { minTime: 0, maxTime: 0, totalDuration: 0 }
+		const minTime = Math.min(...spans.map((s) => s.start_time_unix_nano))
+		const maxTime = Math.max(...spans.map((s) => s.end_time_unix_nano))
+		return { minTime, maxTime, totalDuration: maxTime - minTime }
+	}
+
+	let timelineMetrics = $derived(getTimelineMetrics(traces))
+</script>
+
+<div class="p-4">
+	{#if loading}
+		<Skeleton layout={[[4], [8], [6], [10]]} />
+	{:else if error}
+		<Alert type="error" title="Error">{error}</Alert>
+	{:else if traces.length === 0}
+		<div class="flex flex-col items-center justify-center py-8 text-secondary">
+			<Activity size={48} class="mb-4 opacity-50" />
+			<p class="text-lg font-medium">No traces found</p>
+			<p class="text-sm mt-2">
+				This job did not generate any OTel traces. Make sure OTel auto-instrumentation is enabled
+				and the script uses OTel libraries.
+			</p>
+		</div>
+	{:else}
+		<div class="space-y-4">
+			<div class="flex items-center justify-between">
+				<h3 class="text-lg font-semibold">Traces ({traces.length} spans)</h3>
+				<button
+					class="text-sm text-blue-600 hover:underline"
+					onclick={loadTraces}
+				>
+					Refresh
+				</button>
+			</div>
+
+			<!-- Timeline view -->
+			<div class="border rounded-lg overflow-hidden">
+				<div class="bg-surface-secondary px-4 py-2 border-b">
+					<div class="grid grid-cols-12 gap-2 text-xs font-medium text-secondary">
+						<div class="col-span-4">Operation</div>
+						<div class="col-span-1">Status</div>
+						<div class="col-span-5">Timeline</div>
+						<div class="col-span-2 text-right">Duration</div>
+					</div>
+				</div>
+
+				<div class="divide-y">
+					{#each traces as span (span.span_id)}
+						{@const startOffset =
+							((span.start_time_unix_nano - timelineMetrics.minTime) /
+								timelineMetrics.totalDuration) *
+							100}
+						{@const width = (span.duration_ns / timelineMetrics.totalDuration) * 100}
+
+						<div class="hover:bg-surface-hover">
+							<button
+								class="w-full px-4 py-2 text-left"
+								onclick={() => toggleSpan(span.span_id)}
+							>
+								<div class="grid grid-cols-12 gap-2 items-center">
+									<div class="col-span-4 flex items-center gap-2">
+										<span class="text-xs text-secondary">
+											{expandedSpans.has(span.span_id) ? '▼' : '▶'}
+										</span>
+										<span class="font-medium truncate" title={span.operation_name}>
+											{span.operation_name}
+										</span>
+									</div>
+									<div class="col-span-1">
+										<span class={`text-xs font-medium ${getStatusColor(span.status_code)}`}>
+											{getStatusLabel(span.status_code)}
+										</span>
+									</div>
+									<div class="col-span-5 relative h-4">
+										<div class="absolute inset-0 bg-surface-secondary rounded"></div>
+										<div
+											class="absolute h-full bg-blue-500 rounded opacity-75"
+											style="left: {startOffset}%; width: {Math.max(width, 0.5)}%;"
+										></div>
+									</div>
+									<div class="col-span-2 text-right text-sm font-mono">
+										{formatDuration(span.duration_ns)}
+									</div>
+								</div>
+							</button>
+
+							{#if expandedSpans.has(span.span_id)}
+								<div class="px-4 pb-4 bg-surface-secondary/50">
+									<div class="grid grid-cols-2 gap-4 text-sm">
+										<div>
+											<p class="text-xs text-secondary mb-1">Trace ID</p>
+											<p class="font-mono text-xs break-all">{span.trace_id}</p>
+										</div>
+										<div>
+											<p class="text-xs text-secondary mb-1">Span ID</p>
+											<p class="font-mono text-xs">{span.span_id}</p>
+										</div>
+										{#if span.parent_span_id}
+											<div>
+												<p class="text-xs text-secondary mb-1">Parent Span ID</p>
+												<p class="font-mono text-xs">{span.parent_span_id}</p>
+											</div>
+										{/if}
+										{#if span.service_name}
+											<div>
+												<p class="text-xs text-secondary mb-1">Service</p>
+												<p class="font-mono text-xs">{span.service_name}</p>
+											</div>
+										{/if}
+										<div>
+											<p class="text-xs text-secondary mb-1">Start Time</p>
+											<p class="font-mono text-xs">{formatTimestamp(span.start_time_unix_nano)}</p>
+										</div>
+										<div>
+											<p class="text-xs text-secondary mb-1">End Time</p>
+											<p class="font-mono text-xs">{formatTimestamp(span.end_time_unix_nano)}</p>
+										</div>
+										{#if span.status_message}
+											<div class="col-span-2">
+												<p class="text-xs text-secondary mb-1">Status Message</p>
+												<p class="text-xs">{span.status_message}</p>
+											</div>
+										{/if}
+										{#if Object.keys(span.attributes).length > 0}
+											<div class="col-span-2">
+												<p class="text-xs text-secondary mb-1">Attributes</p>
+												<pre class="text-xs bg-surface-secondary p-2 rounded overflow-x-auto">{JSON.stringify(span.attributes, null, 2)}</pre>
+											</div>
+										{/if}
+										{#if span.events && span.events.length > 0}
+											<div class="col-span-2">
+												<p class="text-xs text-secondary mb-1">Events ({span.events.length})</p>
+												<pre class="text-xs bg-surface-secondary p-2 rounded overflow-x-auto">{JSON.stringify(span.events, null, 2)}</pre>
+											</div>
+										{/if}
+									</div>
+								</div>
+							{/if}
+						</div>
+					{/each}
+				</div>
+			</div>
+		</div>
+	{/if}
+</div>

frontend/src/lib/components/instanceSettings.ts

@@ -34,6 +34,7 @@ export interface Setting {
 		| 'smtp_connect'
 		| 'indexer_rates'
 		| 'otel'
+		| 'otel_auto_instrumentation'
 	storage: SettingStorage
 	advancedToggle?: {
 		label: string
@@ -445,7 +446,15 @@ export const settings: Record<string, Setting[]> = {
 			storage: 'setting',
 			ee_only: ''
 		},
-
+		{
+			label: 'OTel Auto-Instrumentation',
+			description:
+				'Enable automatic OpenTelemetry instrumentation for Python and TypeScript scripts. When enabled, scripts using OTel libraries will automatically send traces to a built-in collector that stores them for viewing in job details.',
+			key: 'otel_auto_instrumentation',
+			fieldType: 'otel_auto_instrumentation',
+			storage: 'setting',
+			ee_only: 'OTel auto-instrumentation is an EE feature'
+		},
 		{
 			label: 'Prometheus',
 			description:

frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte

@@ -91,13 +91,14 @@
 	import RunBadges from '$lib/components/runs/RunBadges.svelte'
 	import { twMerge } from 'tailwind-merge'
 	import FlowRestartButton from '$lib/components/FlowRestartButton.svelte'
+	import JobOtelTraces from '$lib/components/JobOtelTraces.svelte'
 	let job: (Job & { result?: any; result_stream?: string }) | undefined = $state()
 	let jobUpdateLastFetch: Date | undefined = $state()
 
 	let scriptProgress: number | undefined = $state(undefined)
 	let currentJobIsLongRunning: boolean = $state(false)
 
-	let viewTab: 'result' | 'logs' | 'code' | 'stats' | 'assets' = $state('result')
+	let viewTab: 'result' | 'logs' | 'code' | 'stats' | 'assets' | 'traces' = $state('result')
 	let selectedJobStep: string | undefined = $state(undefined)
 
 	let selectedJobStepIsTopLevel: boolean | undefined = $state(undefined)
@@ -769,6 +770,7 @@
 						<Tab value="result" label="Result" />
 						<Tab value="logs" label="Logs" />
 						<Tab value="stats" label="Metrics" />
+						<Tab value="traces" label="Traces" />
 						<Tab value="assets" label="Assets" />
 						{#if isScriptPreview(job?.job_kind)}
 							<Tab value="code" label="Code" />
@@ -798,6 +800,10 @@
 								<div class="w-full">
 									<JobAssetsViewer {job} />
 								</div>
+							{:else if viewTab == 'traces'}
+								<div class="w-full">
+									<JobOtelTraces jobId={job.id} />
+								</div>
 							{:else if viewTab == 'code'}
 								{#if job && 'raw_code' in job && job.raw_code}
 									<div class="text-xs">