/* * Stress test for parallel nativets execution. * * Spawns 8 workers and pushes hundreds of fast nativets jobs concurrently. * Tests: arithmetic, JSON manipulation, string ops, fetch to an internal HTTP server, * and async operations. Validates all jobs complete successfully with correct results. * * IMPORTANT: V8 segfaults when the test harness captures stdout (default behavior). * Always run with --nocapture: * cargo test -p windmill --features "deno_core" --test nativets_stress -- --nocapture */ #[cfg(feature = "deno_core")] use windmill_test_utils::*; #[cfg(feature = "deno_core")] use std::time::Instant; #[cfg(feature = "deno_core")] use futures::StreamExt; #[cfg(feature = "deno_core")] use serde_json::json; #[cfg(feature = "deno_core")] use sqlx::{Pool, Postgres}; #[cfg(feature = "deno_core")] use uuid::Uuid; #[cfg(feature = "deno_core")] use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, worker::{Connection, WORKER_CONFIG}, KillpillSender, }; #[cfg(feature = "deno_core")] use windmill_queue::PushIsolationLevel; #[cfg(feature = "deno_core")] const NUM_WORKERS: usize = 8; /// Various nativets scripts that are fast to execute but cover different features. #[cfg(feature = "deno_core")] fn job_scripts() -> Vec<(&'static str, serde_json::Value, serde_json::Value)> { vec![ // (script_code, args_json, expected_result) ( r#"//native export function main(x: number, y: number): number { return x + y; } "#, json!({"x": 10, "y": 32}), json!(42), ), ( r#"//native export function main(s: string): string { return s.split('').reverse().join(''); } "#, json!({"s": "hello"}), json!("olleh"), ), ( r#"//native export function main(n: number): number[] { return Array.from({length: n}, (_, i) => i * i); } "#, json!({"n": 5}), json!([0, 1, 4, 9, 16]), ), ( r#"//native export function main(items: {name: string, value: number}[]): {total: number, names: string[]} { return { total: items.reduce((sum, i) => sum + i.value, 0), names: items.map(i => i.name).sort(), }; } "#, json!({"items": [{"name": "c", "value": 30}, {"name": "a", "value": 10}, {"name": "b", "value": 20}]}), json!({"total": 60, "names": ["a", "b", "c"]}), ), ( r#"//native export function main(a: number): object { const fib = (n: number): number => n <= 1 ? n : fib(n - 1) + fib(n - 2); return { input: a, fib: fib(a) }; } "#, json!({"a": 10}), json!({"input": 10, "fib": 55}), ), ( r#"//native export function main(text: string, pattern: string): string[] { const regex = new RegExp(pattern, 'g'); return [...text.matchAll(regex)].map(m => m[0]); } "#, json!({"text": "foo123bar456baz789", "pattern": "\\d+"}), json!(["123", "456", "789"]), ), ( r#"//native export function main(obj: Record): Record { return Object.fromEntries( Object.entries(obj).map(([k, v]) => [k.toUpperCase(), String(v * 2)]) ); } "#, json!({"obj": {"a": 1, "b": 2, "c": 3}}), json!({"A": "2", "B": "4", "C": "6"}), ), ( r#"//native export function main(): string { const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } }); const parsed = JSON.parse(data); return parsed.nested.arr.map((x: number) => x * 10).join(","); } "#, json!({}), json!("10,20,30"), ), ] } #[cfg(feature = "deno_core")] async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) -> Uuid { let mut hm_args = std::collections::HashMap::new(); if let Some(obj) = args.as_object() { for (k, v) in obj { hm_args.insert(k.clone(), windmill_common::worker::to_raw_value(v)); } } let job = JobPayload::Code(RawCode { hash: None, content: content.to_string(), path: None, language: ScriptLang::Bun, // //native annotation → nativets lock: None, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, modules: None, }); let tx = PushIsolationLevel::IsolatedRoot(db.clone()); let (uuid, tx) = windmill_queue::push( db, tx, "test-workspace", job, windmill_queue::PushArgs::from(&hm_args), /* user */ "test-user", /* email */ "test@windmill.dev", /* permissioned_as */ "u/test-user".to_string(), /* token_prefix */ None, /* scheduled_for */ None, /* schedule_path */ None, /* parent_job */ None, /* root_job */ None, /* flow_innermost_root_job */ None, /* job_id */ None, /* is_flow_step */ false, /* same_worker */ false, None, true, None, None, None, None, None, false, None, None, None, ) .await .expect("push must succeed"); tx.commit().await.unwrap(); uuid } #[cfg(feature = "deno_core")] fn spawn_workers( conn: &Connection, port: u16, n: usize, ) -> (KillpillSender, Vec>) { use std::sync::atomic::{AtomicUsize, Ordering}; static WORKER_ID: AtomicUsize = AtomicUsize::new(0); std::fs::DirBuilder::new() .recursive(true) .create(&*windmill_worker::GO_BIN_CACHE_DIR) .expect("could not create initial worker dir"); let (tx, _) = KillpillSender::new(n + 1); let mut handles = Vec::with_capacity(n); for i in 0..n { let rx = tx.subscribe(); let conn = conn.clone(); let tx2 = tx.clone(); let id = WORKER_ID.fetch_add(1, Ordering::SeqCst); let worker_name = format!("{id}/stress-w{i}"); let future = async move { let base_internal_url = format!("http://localhost:{}", port); { let mut wc = WORKER_CONFIG.write().await; wc.worker_tags = windmill_common::worker::DEFAULT_TAGS.clone(); wc.priority_tags_sorted = vec![windmill_common::worker::PriorityTags { priority: 0, tags: wc.worker_tags.clone(), }]; windmill_common::worker::store_suspended_pull_query(&wc).await; windmill_common::worker::store_pull_query(&wc).await; } windmill_worker::run_worker( &conn, "test-host", worker_name, i as u64, n as u32, "127.0.0.1", rx, tx2, &base_internal_url, ) .await; }; handles.push(tokio::task::spawn(future)); } (tx, handles) } #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_parallel_nativets_stress(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; set_jwt_secret().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); let conn = Connection::Sql(db.clone()); // Set up completed job listener let mut listener = listen_for_completed_jobs(&db).await; // Prepare job scripts let scripts = job_scripts(); // Also prepare fetch-based scripts that hit the internal API server let fetch_script_template = |port: u16| -> Vec<(String, serde_json::Value)> { vec![ ( format!( r#"//native export async function main(): Promise {{ const resp = await fetch("http://localhost:{port}/api/version"); return {{ status: resp.status, ok: resp.ok }}; }} "# ), json!({}), ), ( format!( r#"//native export async function main(): Promise {{ const resp = await fetch("http://localhost:{port}/api/version"); const text = await resp.text(); return typeof text; }} "# ), json!({}), ), ] }; let fetch_scripts = fetch_script_template(port); let total_jobs = 200; let start = Instant::now(); let mut expected_results: Vec<(Uuid, Option)> = Vec::with_capacity(total_jobs); for i in 0..total_jobs { let (uuid, expected) = if i % 10 < 8 { // 80% non-fetch jobs let idx = i % scripts.len(); let (code, args, expected) = &scripts[idx]; let uuid = push_job(&db, code, args).await; (uuid, Some(expected.clone())) } else { // 20% fetch jobs (we don't check exact result since version string varies) let idx = i % fetch_scripts.len(); let (code, args) = &fetch_scripts[idx]; let uuid = push_job(&db, code, args).await; (uuid, None) // Don't check exact value, just success }; expected_results.push((uuid, expected)); } let push_duration = start.elapsed(); tracing::info!( "Pushed {} jobs in {:?} ({:?}/job)", total_jobs, push_duration, push_duration / total_jobs as u32 ); // Spawn 8 workers let (killpill, worker_handles) = spawn_workers(&conn, port, NUM_WORKERS); // Wait for all jobs to complete let mut completed: std::collections::HashSet = std::collections::HashSet::new(); let timeout_dur = tokio::time::Duration::from_secs(120); let deadline = Instant::now() + timeout_dur; while completed.len() < total_jobs { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { panic!( "Timed out waiting for jobs: {}/{} completed", completed.len(), total_jobs ); } match tokio::time::timeout(remaining, listener.next()).await { Ok(Some(uuid)) => { completed.insert(uuid); if completed.len() % 50 == 0 { tracing::info!( "Progress: {}/{} jobs completed", completed.len(), total_jobs ); } } Ok(None) => panic!("Listener stream ended"), Err(_) => panic!("Timed out: {}/{} completed", completed.len(), total_jobs), } } let exec_duration = start.elapsed(); tracing::info!( "All {} jobs completed in {:?} with {} workers ({:?}/job avg)", total_jobs, exec_duration, NUM_WORKERS, exec_duration / total_jobs as u32 ); // Kill workers killpill.send(); for handle in worker_handles { let _ = tokio::time::timeout(std::time::Duration::from_secs(10), handle).await; } // Verify results let mut successes = 0; let mut failures = 0; for (uuid, expected) in &expected_results { let job = completed_job(*uuid, &db).await; if !job.success { failures += 1; tracing::error!("Job {} FAILED: {:?}", uuid, job.result); continue; } successes += 1; if let Some(expected_val) = expected { let result = job .json_result() .expect("successful job should have result"); assert_eq!( result, *expected_val, "Job {} produced wrong result.\nExpected: {}\nGot: {}", uuid, expected_val, result ); } } tracing::info!( "Results: {} successes, {} failures out of {} total", successes, failures, total_jobs ); assert_eq!(failures, 0, "All jobs should succeed"); assert_eq!(successes, total_jobs, "All jobs should be verified"); Ok(()) }