From 1ceea1c4ffc0ef5bb747008fdcb18c5241d16ece Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 8 Feb 2026 10:16:37 +0100 Subject: [PATCH] refactor: isolate deno_core into windmill-runtime-nativets subcrate (#7848) * refactor: isolate deno_core into windmill-runtime-nativets subcrate Remove deno_core from flow eval and isolate nativets V8 runtime into a dedicated subcrate so deno_core compilation no longer blocks windmill-worker or windmill-api. - Create windmill-jseval crate: QuickJS-based JS eval for flow expressions and batch rerun, extracted from windmill-worker - Create windmill-runtime-nativets crate: all deno_core/V8 deps and nativets script execution, with build.rs snapshot generation - Simplify windmill-worker: remove all deno_* direct deps, empty build.rs, gate nativets behind optional dep - Update windmill-api: use windmill-jseval for batch rerun instead of deno_core, remove deno_core feature entirely - Add nativets integration tests (nativets_jobs.rs) and parallel stress test (nativets_stress.rs, 8 workers x 200 jobs) - Remove dead code: deno flow eval path, USE_QUICKJS env var, parity tests (replaced with 63 standalone expected-value tests) Co-Authored-By: Claude Opus 4.6 * fix: address PR review feedback for deno_core isolation - Deduplicate unsafe_raw() into windmill-common/src/utils.rs (single source) - Delete orphaned runtime.js and windmill-client.js from windmill-worker/src/ - Fix operator precedence in windmill-jseval with explicit parentheses - Remove unnecessary return keyword in heap limit callback - Remove redundant as usize casts - Remove ~150 lines of commented-out code from runtime.js - Remove commented-out #[cfg] in build.rs Co-Authored-By: Claude Opus 4.6 * otel ee --------- Co-authored-by: Claude Opus 4.6 --- backend/Cargo.lock | 79 +- backend/Cargo.toml | 11 +- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 21 +- backend/tests/nativets_jobs.rs | 279 + backend/tests/nativets_stress.rs | 428 ++ backend/windmill-api/Cargo.toml | 5 +- backend/windmill-api/src/jobs.rs | 57 +- backend/windmill-common/src/utils.rs | 8 + backend/windmill-jseval/Cargo.toml | 25 + backend/windmill-jseval/src/lib.rs | 2622 +++++++++ backend/windmill-runtime-nativets/Cargo.toml | 64 + backend/windmill-runtime-nativets/build.rs | 124 + backend/windmill-runtime-nativets/src/lib.rs | 710 +++ .../windmill-runtime-nativets/src/runtime.js | 62 + .../src/windmill-client.js | 0 backend/windmill-worker/Cargo.toml | 37 +- backend/windmill-worker/build.rs | 153 - backend/windmill-worker/src/common.rs | 5 +- backend/windmill-worker/src/js_eval.rs | 1680 +----- .../src/js_eval_parity_tests.rs | 4785 ----------------- .../windmill-worker/src/js_eval_quickjs.rs | 957 ---- backend/windmill-worker/src/lib.rs | 4 - backend/windmill-worker/src/runtime.js | 212 - 24 files changed, 4450 insertions(+), 7880 deletions(-) create mode 100644 backend/tests/nativets_jobs.rs create mode 100644 backend/tests/nativets_stress.rs create mode 100644 backend/windmill-jseval/Cargo.toml create mode 100644 backend/windmill-jseval/src/lib.rs create mode 100644 backend/windmill-runtime-nativets/Cargo.toml create mode 100644 backend/windmill-runtime-nativets/build.rs create mode 100644 backend/windmill-runtime-nativets/src/lib.rs create mode 100644 backend/windmill-runtime-nativets/src/runtime.js rename backend/{windmill-worker => windmill-runtime-nativets}/src/windmill-client.js (100%) delete mode 100644 backend/windmill-worker/src/js_eval_parity_tests.rs delete mode 100644 backend/windmill-worker/src/js_eval_quickjs.rs delete mode 100644 backend/windmill-worker/src/runtime.js diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b3b700c7df..7720be162d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15663,7 +15663,6 @@ dependencies = [ "bitflags 2.9.4", "chrono", "constant_time_eq 0.3.1", - "deno_core", "dotenv", "futures", "gethostname", @@ -15702,7 +15701,6 @@ dependencies = [ "tracing", "url", "uuid", - "v8", "windmill-api", "windmill-api-client", "windmill-autoscaling", @@ -15711,6 +15709,7 @@ dependencies = [ "windmill-git-sync", "windmill-indexer", "windmill-queue", + "windmill-runtime-nativets", "windmill-worker", "windows-service", "windows-sys 0.52.0", @@ -15745,8 +15744,6 @@ dependencies = [ "cron", "dashmap 6.1.0", "datafusion", - "deno_core", - "deno_error", "ed25519-dalek", "flate2", "futures", @@ -15814,6 +15811,7 @@ dependencies = [ "windmill-dep-map", "windmill-git-sync", "windmill-indexer", + "windmill-jseval", "windmill-mcp", "windmill-native-triggers", "windmill-oauth", @@ -16122,6 +16120,23 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-jseval" +version = "1.628.3" +dependencies = [ + "anyhow", + "futures", + "lazy_static", + "mappable-rc", + "regex", + "rquickjs", + "serde_json", + "tokio", + "tracing", + "uuid", + "windmill-common", +] + [[package]] name = "windmill-macros" version = "1.628.3" @@ -16454,6 +16469,43 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-runtime-nativets" +version = "1.628.3" +dependencies = [ + "anyhow", + "const_format", + "deno_ast", + "deno_console", + "deno_core", + "deno_error", + "deno_fetch", + "deno_io", + "deno_net", + "deno_permissions", + "deno_runtime", + "deno_telemetry", + "deno_tls", + "deno_url", + "deno_web", + "deno_webidl", + "futures", + "itertools 0.14.0", + "lazy_static", + "regex", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "uuid", + "winapi", + "windmill-common", + "windmill-parser-ts", + "windmill-queue", +] + [[package]] name = "windmill-sql-datatype-parser-wasm" version = "1.628.3" @@ -16792,20 +16844,6 @@ dependencies = [ "chrono", "const_format", "convert_case 0.6.0", - "deno_ast", - "deno_console", - "deno_core", - "deno_error", - "deno_fetch", - "deno_io", - "deno_net", - "deno_permissions", - "deno_runtime", - "deno_telemetry", - "deno_tls", - "deno_url", - "deno_web", - "deno_webidl", "derive_more 1.0.0", "dotenv", "eventsource-stream", @@ -16844,9 +16882,7 @@ dependencies = [ "regex", "reqwest 0.13.1", "reqwest-middleware", - "rquickjs", "rust_decimal", - "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.9", @@ -16863,11 +16899,11 @@ dependencies = [ "url", "urlencoding", "uuid", - "winapi", "windmill-audit", "windmill-common", "windmill-dep-map", "windmill-git-sync", + "windmill-jseval", "windmill-macros", "windmill-mcp", "windmill-parser", @@ -16886,6 +16922,7 @@ dependencies = [ "windmill-parser-ts", "windmill-parser-yaml", "windmill-queue", + "windmill-runtime-nativets", "yaml-rust", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 860303f5ac..51c195d25b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -28,6 +28,8 @@ members = [ "./windmill-worker", "./windmill-dep-map", "./windmill-common", + "./windmill-jseval", + "./windmill-runtime-nativets", "./windmill-mcp", "./windmill-audit", "./windmill-git-sync", @@ -85,8 +87,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] -deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"] -quickjs = ["windmill-worker/quickjs"] +deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets"] deno_core_mac = ["deno_core", "windmill-worker/libffi_mac"] kafka = ["windmill-api/kafka"] kafka-gssapi = ["windmill-api/kafka-gssapi"] @@ -193,13 +194,12 @@ serde_json.workspace = true serde_derive.workspace = true serde_yml.workspace = true serde.workspace = true -deno_core = { workspace = true, optional = true } +windmill-runtime-nativets = { workspace = true, optional = true } object_store = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } constant_time_eq = { workspace = true, optional = true } quote.workspace = true memchr.workspace = true -v8 = { workspace = true, optional = true } rustls.workspace = true pep440_rs.workspace = true strum.workspace = true @@ -230,7 +230,6 @@ windmill-dep-map.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true -deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } tempfile.workspace = true @@ -278,6 +277,8 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } +windmill-jseval = { path = "./windmill-jseval" } +windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } reqwest-retry = "^0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9c34cb9df3..14252c346d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a69c11d8279401ede1ad5b54e3678c3efb3d2381 \ No newline at end of file +327cf1bff1c5a61f6ea2bd81f1476bee51d152c5 \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 22534f1962..be424a6712 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -127,27 +127,8 @@ mod monitor; mod windows_service_ee; pub fn setup_deno_runtime() -> anyhow::Result<()> { - // https://github.com/denoland/deno/blob/main/cli/main.rs#L477 #[cfg(feature = "deno_core")] - let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ - "--stack-size=1024".to_string(), - // TODO(bartlomieju): I think this can be removed as it's handled by `deno_core` - // and its settings. - // deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript - // TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast - "--no-harmony-import-assertions".to_string(), - ]) - .into_iter() - .skip(1) - .collect::>(); - - #[cfg(feature = "deno_core")] - if !unrecognized_v8_flags.is_empty() { - println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags); - } - - #[cfg(feature = "deno_core")] - deno_core::JsRuntime::init_platform(None, false); + windmill_runtime_nativets::setup_deno_runtime()?; Ok(()) } diff --git a/backend/tests/nativets_jobs.rs b/backend/tests/nativets_jobs.rs new file mode 100644 index 0000000000..a8fbb483ed --- /dev/null +++ b/backend/tests/nativets_jobs.rs @@ -0,0 +1,279 @@ +/* + * Integration tests for nativets (//native) job execution. + * + * These tests ensure that the V8 runtime is properly initialized and that + * nativets jobs execute correctly without segfaults. They cover sync functions, + * async functions, and fetch operations. + * + * All nativets tests run inside a single #[sqlx::test] with a single long-lived + * worker, because V8 isolates cannot be safely created/destroyed/recreated + * across different tokio runtimes or worker lifecycles. + * + * Run with: + * cargo test -p windmill --features "deno_core" --test nativets_jobs -- --nocapture + */ + +#[cfg(feature = "deno_core")] +mod common; + +#[cfg(feature = "deno_core")] +use common::*; + +#[cfg(feature = "deno_core")] +use futures::StreamExt; +#[cfg(feature = "deno_core")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "deno_core")] +use windmill_common::jobs::{JobPayload, RawCode}; +#[cfg(feature = "deno_core")] +use windmill_common::scripts::ScriptLang; + +#[cfg(feature = "deno_core")] +fn init_nativets_runtime() { + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed"); + }); +} + +#[cfg(feature = "deno_core")] +fn nativets_code(content: &str) -> JobPayload { + JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }) +} + +#[cfg(feature = "deno_core")] +async fn push_and_wait( + db: &Pool, + job: RunJob, + listener: &mut (impl StreamExt + Unpin), +) -> CompletedJob { + let uuid = job.push(db).await; + let deadline = std::time::Instant::now() + tokio::time::Duration::from_secs(60); + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + panic!("Timed out waiting for job {uuid}"); + } + match tokio::time::timeout(remaining, listener.next()).await { + Ok(Some(completed_uuid)) if completed_uuid == uuid => break, + Ok(Some(_)) => continue, + Ok(None) => panic!("Listener ended while waiting for {uuid}"), + Err(_) => panic!("Timed out waiting for job {uuid}"), + } + } + completed_job(uuid, db).await +} + +/// All nativets tests share a single worker to avoid V8 isolate lifecycle issues. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_jobs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + init_nativets_runtime(); + set_jwt_secret().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let mut listener = listen_for_completed_jobs(&db).await; + let conn = windmill_common::worker::Connection::Sql(db.clone()); + let (killpill, _worker_handle) = spawn_test_worker(&conn, port); + + // -- simple string return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): string { + return "hello from nativets"; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "simple_return failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!("hello from nativets") + ); + } + + // -- arithmetic with args -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(x: number, y: number): number { + return x + y; +} +"#, + )) + .arg("x", serde_json::json!(17)) + .arg("y", serde_json::json!(25)), + &mut listener, + ) + .await; + assert!(result.success, "with_args failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!(42)); + } + + // -- complex object return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(items: {name: string, value: number}[]): {total: number, names: string[]} { + return { + total: items.reduce((sum, i) => sum + i.value, 0), + names: items.map(i => i.name).sort(), + }; +} +"#, + )) + .arg( + "items", + serde_json::json!([ + {"name": "c", "value": 30}, + {"name": "a", "value": 10}, + {"name": "b", "value": 20} + ]), + ), + &mut listener, + ) + .await; + assert!(result.success, "object_return failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!({"total": 60, "names": ["a", "b", "c"]}) + ); + } + + // -- async function (Promise) -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export async function main(): Promise { + const result = await Promise.resolve(42); + return result; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "async failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!(42)); + } + + // -- fetch hitting the internal API server -- + { + let code = format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + return {{ status: resp.status, ok: resp.ok }}; +}} +"# + ); + let result = push_and_wait(&db, RunJob::from(nativets_code(&code)), &mut listener).await; + assert!(result.success, "fetch failed: {:?}", result.result); + let val = result.json_result().unwrap(); + assert_eq!(val["ok"], serde_json::json!(true)); + assert_eq!(val["status"], serde_json::json!(200)); + } + + // -- regex -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(text: string, pattern: string): string[] { + const regex = new RegExp(pattern, 'g'); + return [...text.matchAll(regex)].map(m => m[0]); +} +"#, + )) + .arg("text", serde_json::json!("foo123bar456baz789")) + .arg("pattern", serde_json::json!("\\d+")), + &mut listener, + ) + .await; + assert!(result.success, "regex failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!(["123", "456", "789"]) + ); + } + + // -- JSON roundtrip -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): string { + const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } }); + const parsed = JSON.parse(data); + return parsed.nested.arr.map((x: number) => x * 10).join(","); +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "json_roundtrip failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!("10,20,30")); + } + + // -- no args, array return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): number[] { + return Array.from({length: 5}, (_, i) => i * i); +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "no_args failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!([0, 1, 4, 9, 16]) + ); + } + + killpill.send(); + Ok(()) +} diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs new file mode 100644 index 0000000000..896b5e8f3d --- /dev/null +++ b/backend/tests/nativets_stress.rs @@ -0,0 +1,428 @@ +/* + * Stress test for parallel nativets execution. + * + * Spawns 8 workers and pushes hundreds of fast nativets jobs concurrently. + * Tests: arithmetic, JSON manipulation, string ops, fetch to an internal HTTP server, + * and async operations. Validates all jobs complete successfully with correct results. + * + * IMPORTANT: V8 segfaults when the test harness captures stdout (default behavior). + * Always run with --nocapture: + * cargo test -p windmill --features "deno_core" --test nativets_stress -- --nocapture + */ + +#[cfg(feature = "deno_core")] +mod common; + +#[cfg(feature = "deno_core")] +use common::*; + +#[cfg(feature = "deno_core")] +use std::time::Instant; + +#[cfg(feature = "deno_core")] +use futures::StreamExt; +#[cfg(feature = "deno_core")] +use serde_json::json; +#[cfg(feature = "deno_core")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "deno_core")] +use uuid::Uuid; + +#[cfg(feature = "deno_core")] +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, + KillpillSender, +}; +#[cfg(feature = "deno_core")] +use windmill_queue::PushIsolationLevel; + +#[cfg(feature = "deno_core")] +const NUM_WORKERS: usize = 8; + +/// Various nativets scripts that are fast to execute but cover different features. +#[cfg(feature = "deno_core")] +fn job_scripts() -> Vec<(&'static str, serde_json::Value, serde_json::Value)> { + vec![ + // (script_code, args_json, expected_result) + ( + r#"//native + +export function main(x: number, y: number): number { + return x + y; +} +"#, + json!({"x": 10, "y": 32}), + json!(42), + ), + ( + r#"//native + +export function main(s: string): string { + return s.split('').reverse().join(''); +} +"#, + json!({"s": "hello"}), + json!("olleh"), + ), + ( + r#"//native + +export function main(n: number): number[] { + return Array.from({length: n}, (_, i) => i * i); +} +"#, + json!({"n": 5}), + json!([0, 1, 4, 9, 16]), + ), + ( + r#"//native + +export function main(items: {name: string, value: number}[]): {total: number, names: string[]} { + return { + total: items.reduce((sum, i) => sum + i.value, 0), + names: items.map(i => i.name).sort(), + }; +} +"#, + json!({"items": [{"name": "c", "value": 30}, {"name": "a", "value": 10}, {"name": "b", "value": 20}]}), + json!({"total": 60, "names": ["a", "b", "c"]}), + ), + ( + r#"//native + +export function main(a: number): object { + const fib = (n: number): number => n <= 1 ? n : fib(n - 1) + fib(n - 2); + return { input: a, fib: fib(a) }; +} +"#, + json!({"a": 10}), + json!({"input": 10, "fib": 55}), + ), + ( + r#"//native + +export function main(text: string, pattern: string): string[] { + const regex = new RegExp(pattern, 'g'); + return [...text.matchAll(regex)].map(m => m[0]); +} +"#, + json!({"text": "foo123bar456baz789", "pattern": "\\d+"}), + json!(["123", "456", "789"]), + ), + ( + r#"//native + +export function main(obj: Record): Record { + return Object.fromEntries( + Object.entries(obj).map(([k, v]) => [k.toUpperCase(), String(v * 2)]) + ); +} +"#, + json!({"obj": {"a": 1, "b": 2, "c": 3}}), + json!({"A": "2", "B": "4", "C": "6"}), + ), + ( + r#"//native + +export function main(): string { + const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } }); + const parsed = JSON.parse(data); + return parsed.nested.arr.map((x: number) => x * 10).join(","); +} +"#, + json!({}), + json!("10,20,30"), + ), + ] +} + +#[cfg(feature = "deno_core")] +async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) -> Uuid { + let mut hm_args = std::collections::HashMap::new(); + if let Some(obj) = args.as_object() { + for (k, v) in obj { + hm_args.insert(k.clone(), windmill_common::worker::to_raw_value(v)); + } + } + + let job = JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language: ScriptLang::Bun, // //native annotation → nativets + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (uuid, tx) = windmill_queue::push( + db, + tx, + "test-workspace", + job, + windmill_queue::PushArgs::from(&hm_args), + /* user */ "test-user", + /* email */ "test@windmill.dev", + /* permissioned_as */ "u/test-user".to_string(), + /* token_prefix */ None, + /* scheduled_for */ None, + /* schedule_path */ None, + /* parent_job */ None, + /* root_job */ None, + /* flow_innermost_root_job */ None, + /* job_id */ None, + /* is_flow_step */ false, + /* same_worker */ false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await + .expect("push must succeed"); + tx.commit().await.unwrap(); + uuid +} + +#[cfg(feature = "deno_core")] +fn spawn_workers( + conn: &Connection, + port: u16, + n: usize, +) -> (KillpillSender, Vec>) { + use std::sync::atomic::{AtomicUsize, Ordering}; + static WORKER_ID: AtomicUsize = AtomicUsize::new(0); + + std::fs::DirBuilder::new() + .recursive(true) + .create(windmill_worker::GO_BIN_CACHE_DIR) + .expect("could not create initial worker dir"); + + let (tx, _) = KillpillSender::new(n + 1); + let mut handles = Vec::with_capacity(n); + + for i in 0..n { + let rx = tx.subscribe(); + let conn = conn.clone(); + let tx2 = tx.clone(); + let id = WORKER_ID.fetch_add(1, Ordering::SeqCst); + let worker_name = format!("{id}/stress-w{i}"); + + let future = async move { + let base_internal_url = format!("http://localhost:{}", port); + { + let mut wc = WORKER_CONFIG.write().await; + wc.worker_tags = windmill_common::worker::DEFAULT_TAGS.clone(); + wc.priority_tags_sorted = vec![windmill_common::worker::PriorityTags { + priority: 0, + tags: wc.worker_tags.clone(), + }]; + windmill_common::worker::store_suspended_pull_query(&wc).await; + windmill_common::worker::store_pull_query(&wc).await; + } + windmill_worker::run_worker( + &conn, + "test-host", + worker_name, + i as u64, + n as u32, + "127.0.0.1", + rx, + tx2, + &base_internal_url, + ) + .await; + }; + + handles.push(tokio::task::spawn(future)); + } + + (tx, handles) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_parallel_nativets_stress(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + + // V8 and rustls must be initialized before any JsRuntime is created + static RUNTIME_INIT: std::sync::Once = std::sync::Once::new(); + RUNTIME_INIT.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed"); + }); + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let conn = Connection::Sql(db.clone()); + + // Set up completed job listener + let mut listener = listen_for_completed_jobs(&db).await; + + // Prepare job scripts + let scripts = job_scripts(); + + // Also prepare fetch-based scripts that hit the internal API server + let fetch_script_template = |port: u16| -> Vec<(String, serde_json::Value)> { + vec![ + ( + format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + return {{ status: resp.status, ok: resp.ok }}; +}} +"# + ), + json!({}), + ), + ( + format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + const text = await resp.text(); + return typeof text; +}} +"# + ), + json!({}), + ), + ] + }; + + let fetch_scripts = fetch_script_template(port); + let total_jobs = 200; + + let start = Instant::now(); + let mut expected_results: Vec<(Uuid, Option)> = + Vec::with_capacity(total_jobs); + + for i in 0..total_jobs { + let (uuid, expected) = if i % 10 < 8 { + // 80% non-fetch jobs + let idx = i % scripts.len(); + let (code, args, expected) = &scripts[idx]; + let uuid = push_job(&db, code, args).await; + (uuid, Some(expected.clone())) + } else { + // 20% fetch jobs (we don't check exact result since version string varies) + let idx = i % fetch_scripts.len(); + let (code, args) = &fetch_scripts[idx]; + let uuid = push_job(&db, code, args).await; + (uuid, None) // Don't check exact value, just success + }; + expected_results.push((uuid, expected)); + } + + let push_duration = start.elapsed(); + tracing::info!( + "Pushed {} jobs in {:?} ({:?}/job)", + total_jobs, + push_duration, + push_duration / total_jobs as u32 + ); + + // Spawn 8 workers + let (killpill, worker_handles) = spawn_workers(&conn, port, NUM_WORKERS); + + // Wait for all jobs to complete + let mut completed: std::collections::HashSet = std::collections::HashSet::new(); + let timeout_dur = tokio::time::Duration::from_secs(120); + let deadline = Instant::now() + timeout_dur; + + while completed.len() < total_jobs { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + panic!( + "Timed out waiting for jobs: {}/{} completed", + completed.len(), + total_jobs + ); + } + + match tokio::time::timeout(remaining, listener.next()).await { + Ok(Some(uuid)) => { + completed.insert(uuid); + if completed.len() % 50 == 0 { + tracing::info!( + "Progress: {}/{} jobs completed", + completed.len(), + total_jobs + ); + } + } + Ok(None) => panic!("Listener stream ended"), + Err(_) => panic!("Timed out: {}/{} completed", completed.len(), total_jobs), + } + } + + let exec_duration = start.elapsed(); + tracing::info!( + "All {} jobs completed in {:?} with {} workers ({:?}/job avg)", + total_jobs, + exec_duration, + NUM_WORKERS, + exec_duration / total_jobs as u32 + ); + + // Kill workers + killpill.send(); + for handle in worker_handles { + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), handle).await; + } + + // Verify results + let mut successes = 0; + let mut failures = 0; + + for (uuid, expected) in &expected_results { + let job = completed_job(*uuid, &db).await; + if !job.success { + failures += 1; + tracing::error!("Job {} FAILED: {:?}", uuid, job.result); + continue; + } + successes += 1; + + if let Some(expected_val) = expected { + let result = job + .json_result() + .expect("successful job should have result"); + assert_eq!( + result, *expected_val, + "Job {} produced wrong result.\nExpected: {}\nGot: {}", + uuid, expected_val, result + ); + } + } + + tracing::info!( + "Results: {} successes, {} failures out of {} total", + successes, + failures, + total_jobs + ); + + assert_eq!(failures, 0, "All jobs should succeed"); + assert_eq!(successes, total_jobs, "All jobs should be verified"); + + Ok(()) +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 5ef0e43461..08c6164d29 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -36,7 +36,6 @@ postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-store/postgres_tr mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-store/mqtt_trigger"] native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "dep:strum", "oauth2"] sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-store/sqs_trigger"] -deno_core = ["dep:deno_core", "dep:deno_error"] gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-store/gcp_trigger"] cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud"] mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"] @@ -155,12 +154,10 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types.workspace = true async-trait.workspace = true -deno_error = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } +windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true strum = { workspace = true, optional = true } dashmap.workspace = true [build-dependencies] -deno_core = { workspace = true, optional = true } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 667204e7e9..7fe674b2ce 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -12,8 +12,6 @@ pub use windmill_api_jobs::types::*; pub use windmill_api_sse::*; use axum::body::Body; -#[cfg(feature = "deno_core")] -use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; use futures::{StreamExt, TryFutureExt}; use itertools::Itertools; use quick_cache::sync::Cache; @@ -2928,50 +2926,18 @@ struct BatchReRunQueryReturnType { schema: Option, } -#[cfg(feature = "deno_core")] -#[op2] -#[string] -fn get_deno_core_job_value(state: &mut OpState) -> Option { - let obj = state.borrow::(); - let str = serde_json::to_string(&obj).ok()?; - Some(str) -} - -#[cfg(feature = "deno_core")] async fn batch_rerun_compute_js_expression( expr: String, job: BatchReRunQueryReturnType, ) -> error::Result> { - let ext = deno_core::Extension { - name: "batch_rerun_arg_transform_ext", - ops: vec![get_deno_core_job_value()].into(), - ..Default::default() - }; - let mut isolate = - JsRuntime::new(deno_core::RuntimeOptions { extensions: vec![ext], ..Default::default() }); - - { - let op_state = isolate.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(BatchReRunQueryReturnType { schema: None, ..job }); - } - isolate - .execute_script( - "", - "let job = JSON.parse(Deno.core.ops.get_deno_core_job_value());", - ) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - - // Run user expr - let result = isolate - .execute_script("", expr) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let mut scope = isolate.handle_scope(); - let result = v8::Local::new(&mut scope, result); - let result: serde_json::Value = - serde_v8::from_v8(&mut scope, result).map_err(|e| Error::ExecutionErr(e.to_string()))?; - let result = JsonRawValue::from_string(result.to_string())?; - Ok(result) + let job_no_schema = BatchReRunQueryReturnType { schema: None, ..job }; + let job_value = + serde_json::to_value(&job_no_schema).map_err(|e| Error::ExecutionErr(e.to_string()))?; + let mut globals = std::collections::HashMap::new(); + globals.insert("job".to_string(), job_value); + windmill_jseval::eval_simple_js(expr, globals) + .await + .map_err(|e| Error::ExecutionErr(e.to_string())) } async fn batch_rerun_jobs( @@ -3098,13 +3064,6 @@ async fn batch_rerun_handle_job( args.insert(property_name.clone(), value.clone()); } InputTransform::Javascript { expr } => { - #[cfg(not(feature = "deno_core"))] - Err(error::Error::ExecutionErr( - format!("deno_core feature is not activated, cannot evaluate: {expr}") - .to_string(), - ))?; - - #[cfg(feature = "deno_core")] args.insert( property_name.clone(), batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?, diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 8a52d34669..67cac0cbf1 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -974,6 +974,14 @@ pub async fn get_custom_pg_instance_password(db: &DB) -> Result { ) } +/// Convert a JSON string to a `Box` without validation. +/// +/// # Safety +/// The caller must ensure the string is valid JSON. +pub fn unsafe_raw(json: String) -> Box { + unsafe { std::mem::transmute::, Box>(json.into()) } +} + // Avoid JSON parsing for merging raw JSON values into an object pub fn merge_raw_values_to_object( pairs: &[(String, Box)], diff --git a/backend/windmill-jseval/Cargo.toml b/backend/windmill-jseval/Cargo.toml new file mode 100644 index 0000000000..076f1329f3 --- /dev/null +++ b/backend/windmill-jseval/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "windmill-jseval" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_jseval" +path = "src/lib.rs" + +[features] +default = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +rquickjs.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +anyhow.workspace = true +regex.workspace = true +lazy_static.workspace = true +mappable-rc.workspace = true +futures.workspace = true +uuid.workspace = true diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs new file mode 100644 index 0000000000..34fdb62ee5 --- /dev/null +++ b/backend/windmill-jseval/src/lib.rs @@ -0,0 +1,2622 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! QuickJS-based JavaScript expression evaluation for flow transformations. +//! +//! This crate provides fast, lightweight JS expression evaluation using QuickJS (rquickjs). +//! It is used by both windmill-worker (for flow expression eval) and windmill-api (for batch rerun). +//! +//! ## Performance Characteristics (release mode benchmarks) +//! - **Simple expressions**: ~238μs (QuickJS) vs ~3.05ms (deno_core) = **~13x faster** +//! - **Complex expressions**: ~192μs (QuickJS) vs ~3.09ms (deno_core) = **~16x faster** +//! - **Memory**: ~2.5% of V8's footprint + +use std::collections::HashMap; +use std::sync::Arc; + +use lazy_static::lazy_static; +use regex::Regex; +use rquickjs::{ + async_with, + prelude::{Async, Func, MutFn}, + AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object, Value, +}; +use serde_json::value::RawValue; +use uuid::Uuid; + +use windmill_common::client::AuthedClient; +use windmill_common::flow_status::JobResult; + +// ── Public types ────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub struct IdContext { + pub flow_job: Uuid, + #[allow(dead_code)] + pub steps_results: HashMap, + pub previous_id: String, +} + +// ── Constants ───────────────────────────────────────────────────────── + +const FLOW_INPUT_PREFIX: &str = "flow_input"; +const ENV_KEY_PREFIX: &str = "flow_env"; +const DOT_PATTERN: &str = "."; +const START_BRACKET_PATTERN: &str = "[\""; +const END_BRACKET_PATTERN: &str = "\"]"; + +// ── Regex statics ───────────────────────────────────────────────────── + +lazy_static! { + static ref RE: Regex = Regex::new( + r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + ) + .unwrap(); + static ref RE_FULL: Regex = Regex::new( + r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" + ) + .unwrap(); +} + +// ── Shared helper functions ─────────────────────────────────────────── + +pub fn replace_with_await(expr: String, fn_name: &str) -> String { + let sep = format!("{}(", fn_name); + let mut split = expr.split(&sep); + let mut s = split.next().unwrap_or("").to_string(); + for x in split { + s.push_str(&format!("(await {}({}", fn_name, add_closing_bracket(x))) + } + s +} + +pub fn replace_with_await_result(expr: String) -> String { + RE.replace_all(&expr, "(await $r)").to_string() +} + +fn add_closing_bracket(s: &str) -> String { + let mut s = s.to_string(); + let mut level = 1; + let mut idx = 0; + for c in s.chars() { + match c { + '(' => level += 1, + ')' => level -= 1, + _ => (), + }; + if level == 0 { + break; + } + idx += 1; + } + s.insert_str(idx, ")"); + s +} + +pub fn try_exact_property_access( + expr: &str, + flow_input: Option<&mappable_rc::Marc>>>, + flow_env: Option<&HashMap>>, +) -> Option> { + let obj = if expr.starts_with(FLOW_INPUT_PREFIX) { + Some(( + FLOW_INPUT_PREFIX, + flow_input.as_ref().map(|obj| obj.as_ref()), + )) + } else if expr.starts_with(ENV_KEY_PREFIX) { + Some((ENV_KEY_PREFIX, flow_env)) + } else { + None + }; + + if let Some((prefix, obj)) = obj { + let access_pattern_pos = prefix.len(); + let suffix = &expr[access_pattern_pos..]; + let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { + let key_name_pos = DOT_PATTERN.len(); + Some(&suffix[key_name_pos..]) + } else if suffix.starts_with(START_BRACKET_PATTERN) { + let key_name_pos = START_BRACKET_PATTERN.len(); + let suffix = &suffix[key_name_pos..]; + + let flow_arg_name = suffix + .ends_with(END_BRACKET_PATTERN) + .then(|| { + let start_key_name_pos = access_pattern_pos + key_name_pos; + let end_key_name_pos = expr.len() - END_BRACKET_PATTERN.len(); + &expr[start_key_name_pos..end_key_name_pos] + }) + .filter(|s| s.len() > 0); + flow_arg_name + } else { + None + }; + + if let Some(key_name) = maybe_key_name { + if let Some(key_value) = obj.and_then(|obj| obj.get(key_name)) { + return Some(key_value.clone()); + } + } + } + None +} + +pub async fn handle_full_regex( + expr: &str, + authed_client: &AuthedClient, + by_id: &IdContext, +) -> Option>> { + if let Some(captures) = RE_FULL.captures(&expr) { + let obj_name = captures.get(1).unwrap().as_str(); + let obj_key = captures.get(2).unwrap().as_str(); + let idx_o = captures.get(3).map(|y| y.as_str()); + let rest = captures.get(4).map(|y| y.as_str()); + let query = if let Some(idx) = idx_o { + match rest { + Some(rest) => Some(format!("{}{}", idx, rest)), + None => Some(idx.to_string()), + } + } else { + rest.map(|x| x.trim_start_matches('.').to_string()) + }; + + let result = if obj_name == "results" { + let res = authed_client + .get_result_by_id::>>( + &by_id.flow_job.to_string(), + obj_key, + query, + ) + .await + .ok() + .flatten(); + match res { + Some(v) => Ok(v), + None => serde_json::value::to_raw_value(&serde_json::Value::Null) + .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), + } + } else if obj_name == "flow_env" { + authed_client + .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else { + unreachable!(); + }; + + return Some(result); + } + + return None; +} + +use windmill_common::utils::unsafe_raw; + +// ── QuickJS evaluation ─────────────────────────────────────────────── + +/// Shared state for async operations within QuickJS +#[derive(Clone)] +struct AsyncOpState { + client: AuthedClient, +} + +/// Evaluates a JavaScript expression using QuickJS runtime. +/// +/// This function provides flow expression evaluation using QuickJS +/// instead of deno_core/V8 for significantly faster startup times. +/// +/// Unlike deno_core, this uses true async Rust callbacks for `variable()`, +/// `resource()`, and `results.xxx` access - no pre-fetching required. +pub async fn eval_timeout_quickjs( + expr: String, + transform_context: HashMap>>, + flow_input: Option>>>, + flow_env: Option<&HashMap>>, + authed_client: Option<&AuthedClient>, + by_id: Option<&IdContext>, + ctx: Option>, +) -> anyhow::Result> { + let expr = expr.trim().to_string(); + + tracing::debug!( + "evaluating js eval (quickjs): {} with context {:?}", + expr, + transform_context + ); + + // Clone data for the blocking task + let by_id_clone = by_id.cloned(); + let flow_input_clone = flow_input.clone(); + let flow_env_clone = flow_env.cloned(); + let authed_client_clone = authed_client.cloned(); + + // Determine which context keys are actually used in the expression + let p_ids = by_id.map(|x| { + [ + format!("results.{}", x.previous_id), + format!("results?.{}", x.previous_id), + format!("results[\"{}\"]", x.previous_id), + format!("results?.[\"{}\"]", x.previous_id), + ] + }); + + let mut context_keys: Vec = transform_context + .keys() + .filter(|x| expr.contains(&x.to_string())) + .cloned() + .collect(); + + if (!context_keys.contains(&"previous_result".to_string()) + && p_ids.is_some() + && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) + || expr.contains("error") + { + context_keys.push("previous_result".to_string()); + } + + let has_flow_input = expr.contains("flow_input"); + if has_flow_input { + context_keys.push("flow_input".to_string()) + } + + // Filter transform_context to only include used keys + let filtered_context: HashMap>> = transform_context + .into_iter() + .filter(|(k, _)| context_keys.contains(k)) + .collect(); + + let expr_clone = expr.clone(); + + // Run the QuickJS evaluation with a timeout + tokio::time::timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + // Create a new tokio runtime for async operations within the blocking context + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async move { + eval_quickjs_inner( + &expr_clone, + filtered_context, + flow_input_clone, + flow_env_clone, + authed_client_clone, + by_id_clone, + ctx, + context_keys, + ) + .await + }) + }), + ) + .await + .map_err(|_| { + anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)") + })?? +} + +/// Memory limit for QuickJS runtime (32MB). +const QUICKJS_MEMORY_LIMIT: usize = 32 * 1024 * 1024; + +async fn eval_quickjs_inner( + expr: &str, + transform_context: HashMap>>, + flow_input: Option>>>, + flow_env: Option>>, + authed_client: Option, + by_id: Option, + extra_ctx: Option>, + context_keys: Vec, +) -> anyhow::Result> { + let runtime = AsyncRuntime::new()?; + runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; + let context = AsyncContext::full(&runtime).await?; + + // Create shared state for async ops if we have a client + let op_state = authed_client.map(|client| Arc::new(AsyncOpState { client })); + + let op_state_clone = op_state.clone(); + let by_id_clone = by_id.clone(); + + // Transform expression to add await for variable/resource/results access + let expr_with_funcs = ["variable", "resource"] + .into_iter() + .fold(expr.to_string(), replace_with_await); + let transformed_expr = replace_with_await_result(expr_with_funcs); + + async_with!(context => |ctx| { + let globals = ctx.globals(); + + // Set up context variables + for key in &context_keys { + if key == "flow_input" { + if let Some(ref fi) = flow_input { + let json_str = serde_json::to_string(fi.as_ref())?; + let val: serde_json::Value = serde_json::from_str(&json_str)?; + let js_val = json_to_js(&ctx, &val)?; + globals.set(key.as_str(), js_val)?; + } else { + globals.set(key.as_str(), Value::new_null(ctx.clone()))?; + } + } else if let Some(raw_val) = transform_context.get(key) { + let val: serde_json::Value = serde_json::from_str(raw_val.get())?; + let js_val = json_to_js(&ctx, &val)?; + globals.set(key.as_str(), js_val)?; + } + } + + // Set up flow_env if referenced + if expr.contains("flow_env") { + if let Some(ref fe) = flow_env { + let obj = Object::new(ctx.clone())?; + for (k, v) in fe { + let val: serde_json::Value = serde_json::from_str(v.get())?; + let js_val = json_to_js(&ctx, &val)?; + obj.set(k.as_str(), js_val)?; + } + globals.set("flow_env", obj)?; + } else { + globals.set("flow_env", Object::new(ctx.clone())?)?; + } + } + + // Set up additional context variables + if let Some(ctx_vars) = extra_ctx { + for (k, v) in ctx_vars { + globals.set(k.as_str(), v.as_str())?; + } + } + + // Set up error extraction if needed + if expr.contains("error") && context_keys.contains(&"previous_result".to_string()) { + let error_setup = r#" + let error = previous_result?.error; + if (!error) { + if (Array.isArray(previous_result)) { + const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); + if (errors.length === 1) { + error = errors[0].error; + } else if (errors.length > 1) { + error = { + name: 'MultipleErrors', + message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), + errors: previous_result + }; + } else { + error = { + name: 'MultipleErrors', + message: "Could not parse errors", + errors: previous_result + }; + } + } else { + if (previous_result) { + error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; + } else { + error = { name: 'UnknownError', message: 'No error found' }; + } + } + } + "#; + ctx.eval::<(), _>(error_setup).catch(&ctx).map_err(quickjs_error_to_anyhow)?; + } + + // Set up async functions if we have a client + if let Some(ref state) = op_state_clone { + setup_async_ops(&ctx, &globals, state.clone())?; + } else { + // Set up stub functions that throw errors + setup_stub_functions(&ctx, &globals)?; + } + + // Set up results proxy if we have by_id context + if let Some(ref by_id) = by_id_clone { + setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?; + } + + // Determine if we need to add return statement. + let code = if should_add_return_quickjs(&transformed_expr) { + format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) + } else { + format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) + }; + + // Evaluate the expression (returns a Promise that resolves to a JSON string) + let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?; + + // Await the promise + let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?; + + let json_str = String::from_js(&ctx, result) + .unwrap_or_else(|_| "null".to_string()); + + Ok(unsafe_raw(json_str)) + }) + .await +} + +/// Set up async variable() and resource() functions using true Rust async callbacks. +fn setup_async_ops<'js>( + ctx: &rquickjs::Ctx<'js>, + globals: &Object<'js>, + state: Arc, +) -> anyhow::Result<()> { + const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; + + let state_for_var = state.clone(); + globals.set( + "__fetchVariable", + Func::from(Async(MutFn::new(move |path: String| { + let client = state_for_var.client.clone(); + async move { + match client.get_variable_value(&path).await { + Ok(value) => value, + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let state_for_res = state.clone(); + globals.set( + "__fetchResource", + Func::from(Async(MutFn::new(move |path: String| { + let client = state_for_res.client.clone(); + async move { + match client + .get_resource_value_interpolated::(&path, None) + .await + { + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let wrapper_code = r#" + const __ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; + + async function variable(path) { + const result = await __fetchVariable(path); + if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { + throw new Error(result.substring(__ERR_PREFIX.length)); + } + return result; + } + + async function resource(path) { + const result = await __fetchResource(path); + if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { + throw new Error(result.substring(__ERR_PREFIX.length)); + } + return JSON.parse(result); + } + "#; + + ctx.eval::<(), _>(wrapper_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +/// Set up stub functions that throw errors when no client is available +fn setup_stub_functions<'js>( + ctx: &rquickjs::Ctx<'js>, + _globals: &Object<'js>, +) -> anyhow::Result<()> { + let setup_code = r#" + function variable(path) { + return Promise.reject(new Error(`variable() is not available without an authenticated client`)); + } + + function resource(path) { + return Promise.reject(new Error(`resource() is not available without an authenticated client`)); + } + "#; + + ctx.eval::<(), _>(setup_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +/// Set up the `results` Proxy object with dynamic access to step results. +fn setup_results_proxy<'js>( + ctx: &rquickjs::Ctx<'js>, + globals: &Object<'js>, + by_id: &IdContext, + op_state: Option>, +) -> anyhow::Result<()> { + globals.set("__previous_id", by_id.previous_id.clone())?; + + if let Some(state) = op_state { + let by_id_for_result = by_id.clone(); + globals.set( + "__fetchResult", + Func::from(Async(MutFn::new(move |step_id: String| { + let client = state.client.clone(); + let by_id = by_id_for_result.clone(); + let step_id_clone = step_id.clone(); + + let job_result = by_id.steps_results.get(&step_id).cloned(); + let flow_job_id = by_id.flow_job.to_string(); + + async move { + const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; + + let result: Result = match job_result { + Some(jr) => match jr { + JobResult::SingleJob(job_id) => client + .get_completed_job_result::( + &job_id.to_string(), + None, + ) + .await + .map_err(|e| { + format!( + "Failed to fetch result for step '{}': {}", + step_id_clone, e + ) + }), + JobResult::ListJob(job_ids) => { + let futs = job_ids.iter().map(|job_id| { + let client = client.clone(); + let job_id_str = job_id.to_string(); + async move { + client + .get_completed_job_result::( + &job_id_str, + None, + ) + .await + } + }); + let results: Vec<_> = futures::future::join_all(futs).await; + let collected: Result, _> = results.into_iter().collect(); + collected.map(serde_json::Value::Array).map_err(|e| { + format!( + "Failed to fetch results for step '{}': {}", + step_id_clone, e + ) + }) + } + }, + None => Ok(client + .get_result_by_id::( + &flow_job_id, + &step_id_clone, + None, + ) + .await + .ok() + .unwrap_or(serde_json::Value::Null)), + }; + + match result { + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let wrapper_code = r#" + const __RESULT_ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; + async function __getResult(stepId) { + const result = await __fetchResult(stepId); + if (typeof result === 'string' && result.startsWith(__RESULT_ERR_PREFIX)) { + throw new Error(result.substring(__RESULT_ERR_PREFIX.length)); + } + return JSON.parse(result); + } + "#; + ctx.eval::<(), _>(wrapper_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + } else { + let stub_code = r#" + function __getResult(stepId) { + return Promise.reject(new Error('Result fetching not available without authenticated client')); + } + "#; + ctx.eval::<(), _>(stub_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + } + + let proxy_setup = r#" + const results = new Proxy({}, { + get: function(target, name, receiver) { + if (typeof name === 'symbol') { + return undefined; + } + if (name === __previous_id && typeof previous_result !== 'undefined') { + return Promise.resolve(previous_result); + } + return __getResult(name); + } + }); + "#; + ctx.eval::<(), _>(proxy_setup) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +/// Convert a serde_json::Value to a QuickJS Value +fn json_to_js<'js>( + ctx: &rquickjs::Ctx<'js>, + val: &serde_json::Value, +) -> rquickjs::Result> { + match val { + serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), + serde_json::Value::Bool(b) => Ok(Value::new_bool(ctx.clone(), *b)), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + if i >= i32::MIN as i64 && i <= i32::MAX as i64 { + Ok(Value::new_int(ctx.clone(), i as i32)) + } else { + Ok(Value::new_float(ctx.clone(), i as f64)) + } + } else if let Some(f) = n.as_f64() { + Ok(Value::new_float(ctx.clone(), f)) + } else { + Ok(Value::new_float(ctx.clone(), 0.0)) + } + } + serde_json::Value::String(s) => s.clone().into_js(ctx), + serde_json::Value::Array(arr) => { + let js_arr = rquickjs::Array::new(ctx.clone())?; + for (i, item) in arr.iter().enumerate() { + js_arr.set(i, json_to_js(ctx, item)?)?; + } + Ok(js_arr.into_value()) + } + serde_json::Value::Object(obj) => { + let js_obj = Object::new(ctx.clone())?; + for (k, v) in obj { + js_obj.set(k.as_str(), json_to_js(ctx, v)?)?; + } + Ok(js_obj.into_value()) + } + } +} + +fn should_add_return_quickjs(expr: &str) -> bool { + let trimmed = expr.trim(); + + if trimmed.is_empty() { + return true; + } + + if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { + return false; + } + + let statement_prefixes = [ + "const ", + "let ", + "var ", + "if ", + "if(", + "for ", + "for(", + "while ", + "while(", + "switch ", + "switch(", + "try ", + "try{", + "throw ", + "function ", + "class ", + "async ", + "await ", + ]; + + for prefix in &statement_prefixes { + if trimmed.starts_with(prefix) { + return false; + } + } + + if contains_semicolon_outside_strings(trimmed) { + return false; + } + + true +} + +fn contains_semicolon_outside_strings(expr: &str) -> bool { + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut in_template = false; + let mut prev_char = '\0'; + + for ch in expr.chars() { + match ch { + '\'' if prev_char != '\\' && !in_double_quote && !in_template => { + in_single_quote = !in_single_quote; + } + '"' if prev_char != '\\' && !in_single_quote && !in_template => { + in_double_quote = !in_double_quote; + } + '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { + in_template = !in_template; + } + ';' if !in_single_quote && !in_double_quote && !in_template => { + return true; + } + _ => {} + } + prev_char = ch; + } + + false +} + +fn quickjs_error_to_anyhow(err: rquickjs::CaughtError<'_>) -> anyhow::Error { + anyhow::anyhow!("QuickJS evaluation error: {}", err) +} + +// ── eval_simple_js for windmill-api batch rerun ────────────────────── + +/// Evaluate a JS expression with named JSON globals in scope. +/// Used by windmill-api for batch rerun arg transforms. +pub async fn eval_simple_js( + expr: String, + globals: HashMap, +) -> anyhow::Result> { + tokio::time::timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + rt.block_on(async move { + let runtime = AsyncRuntime::new()?; + runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; + let context = AsyncContext::full(&runtime).await?; + + async_with!(context => |ctx| { + let js_globals = ctx.globals(); + + // Set up each named global + for (name, value) in &globals { + let js_val = json_to_js(&ctx, value)?; + js_globals.set(name.as_str(), js_val)?; + } + + // Wrap expression to return JSON string + let code = format!("JSON.stringify(({}) ?? null)", expr); + let result: String = ctx.eval(code) + .catch(&ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(unsafe_raw(result)) + }) + .await + }) + }), + ) + .await + .map_err(|_| { + anyhow::anyhow!("The expression evaluation took too long to execute (>10000ms)") + })?? +} + +// ── Tests ──────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use windmill_common::worker::to_raw_value; + + /// Helper: evaluate an expression with a transform context and assert against expected JSON. + async fn assert_eval( + expr: &str, + ctx: HashMap>>, + expected: serde_json::Value, + ) { + assert_eval_full(expr, ctx, None, None, expected).await; + } + + /// Helper: evaluate with full context (transform_context, flow_input, flow_env). + async fn assert_eval_full( + expr: &str, + ctx: HashMap>>, + flow_input: Option>>>, + flow_env: Option<&HashMap>>, + expected: serde_json::Value, + ) { + let result = eval_timeout_quickjs( + expr.to_string(), + ctx, + flow_input, + flow_env, + None, + None, + None, + ) + .await + .unwrap_or_else(|e| panic!("eval_timeout_quickjs failed for '{}': {}", expr, e)); + + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap_or_else(|e| { + panic!( + "Failed to parse result '{}' for '{}': {}", + result.get(), + expr, + e + ) + }); + + assert_eq!(actual, expected, "Mismatch for expression '{}'", expr); + } + + // ===================================================================== + // HELPER FUNCTION TESTS + // ===================================================================== + + #[test] + fn test_should_add_return_quickjs() { + assert!(should_add_return_quickjs("5")); + assert!(should_add_return_quickjs("x + y")); + assert!(should_add_return_quickjs("foo()")); + assert!(should_add_return_quickjs("obj.method()")); + assert!(should_add_return_quickjs("a > b ? 'yes' : 'no'")); + assert!(should_add_return_quickjs("({ key: 'value' })")); + + assert!(!should_add_return_quickjs("return 5")); + assert!(!should_add_return_quickjs("return x + y")); + assert!(!should_add_return_quickjs("const x = 5")); + assert!(!should_add_return_quickjs("let y = 10")); + assert!(!should_add_return_quickjs("if (x > 5) { return x; }")); + assert!(!should_add_return_quickjs("let x = 5; x + 1")); + assert!(!should_add_return_quickjs("try { return 1; } catch(e) {}")); + } + + #[test] + fn test_contains_semicolon_outside_strings() { + assert!(contains_semicolon_outside_strings("a; b")); + assert!(contains_semicolon_outside_strings("let x = 5; x + 1")); + + assert!(!contains_semicolon_outside_strings("'a;b'")); + assert!(!contains_semicolon_outside_strings("\"a;b\"")); + assert!(!contains_semicolon_outside_strings("`a;b`")); + assert!(!contains_semicolon_outside_strings("x + y")); + } + + #[test] + fn test_replace_with_await() { + assert_eq!( + replace_with_await("variable('test')".to_string(), "variable"), + "(await variable('test'))" + ); + assert_eq!( + replace_with_await("x + variable('a') + variable('b')".to_string(), "variable"), + "x + (await variable('a')) + (await variable('b'))" + ); + assert_eq!( + replace_with_await("no_match".to_string(), "variable"), + "no_match" + ); + } + + #[test] + fn test_replace_with_await_result() { + assert_eq!( + replace_with_await_result("results.step_a".to_string()), + "(await results.step_a)" + ); + assert_eq!( + replace_with_await_result("results.a + results.b".to_string()), + "(await results.a) + (await results.b)" + ); + assert_eq!( + replace_with_await_result("no_results_here".to_string()), + "no_results_here" + ); + } + + #[test] + fn test_try_exact_property_access_flow_input_dot() { + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("hello"))); + let fi = mappable_rc::Marc::new(fi); + + let result = try_exact_property_access("flow_input.name", Some(&fi), None); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "\"hello\""); + } + + #[test] + fn test_try_exact_property_access_flow_input_bracket() { + let mut fi = HashMap::new(); + fi.insert("my_key".to_string(), to_raw_value(&json!(42))); + let fi = mappable_rc::Marc::new(fi); + + let result = try_exact_property_access("flow_input[\"my_key\"]", Some(&fi), None); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "42"); + } + + #[test] + fn test_try_exact_property_access_flow_env() { + let mut fe = HashMap::new(); + fe.insert("ENV".to_string(), to_raw_value(&json!("production"))); + + let result = try_exact_property_access("flow_env.ENV", None, Some(&fe)); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "\"production\""); + } + + #[test] + fn test_try_exact_property_access_missing_key() { + let fi = mappable_rc::Marc::new(HashMap::new()); + let result = try_exact_property_access("flow_input.missing", Some(&fi), None); + assert!(result.is_none()); + } + + #[test] + fn test_try_exact_property_access_no_prefix() { + let result = try_exact_property_access("some_var.key", None, None); + assert!(result.is_none()); + } + + // ===================================================================== + // SIMPLE ARITHMETIC + // ===================================================================== + + #[tokio::test] + async fn test_simple_arithmetic() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); + + assert_eval("x + y", env.clone(), json!(8)).await; + assert_eval("x - y", env.clone(), json!(2)).await; + assert_eval("x * y", env.clone(), json!(15)).await; + assert_eval("x % y", env.clone(), json!(2)).await; + assert_eval("x ** 2", env.clone(), json!(25)).await; + } + + // ===================================================================== + // OBJECT PROPERTY ACCESS + // ===================================================================== + + #[tokio::test] + async fn test_object_property_access() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({ + "name": "test", + "value": 42, + "nested": {"deep": {"property": "found"}} + }))), + ); + + assert_eval("obj.name", env.clone(), json!("test")).await; + assert_eval("obj.value", env.clone(), json!(42)).await; + assert_eval("obj.nested.deep.property", env.clone(), json!("found")).await; + assert_eval("obj['name']", env.clone(), json!("test")).await; + } + + // ===================================================================== + // ARRAY OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_array_operations() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), + ); + + assert_eval("arr.length", env.clone(), json!(5)).await; + assert_eval("arr[0]", env.clone(), json!(1)).await; + assert_eval("arr.map(x => x * 2)", env.clone(), json!([2, 4, 6, 8, 10])).await; + assert_eval("arr.filter(x => x > 2)", env.clone(), json!([3, 4, 5])).await; + assert_eval("arr.reduce((a, b) => a + b, 0)", env.clone(), json!(15)).await; + assert_eval("arr.find(x => x > 3)", env.clone(), json!(4)).await; + assert_eval("arr.some(x => x > 4)", env.clone(), json!(true)).await; + assert_eval("arr.every(x => x > 0)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // STRING OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_string_operations() { + let mut env = HashMap::new(); + env.insert( + "s".to_string(), + Arc::new(to_raw_value(&json!("Hello World"))), + ); + + assert_eval("s.toLowerCase()", env.clone(), json!("hello world")).await; + assert_eval("s.toUpperCase()", env.clone(), json!("HELLO WORLD")).await; + assert_eval("s.length", env.clone(), json!(11)).await; + assert_eval("s.split(' ')", env.clone(), json!(["Hello", "World"])).await; + assert_eval( + "s.replace('World', 'QuickJS')", + env.clone(), + json!("Hello QuickJS"), + ) + .await; + assert_eval("s.includes('World')", env.clone(), json!(true)).await; + assert_eval("s.startsWith('Hello')", env.clone(), json!(true)).await; + assert_eval("s.trim()", env.clone(), json!("Hello World")).await; + } + + // ===================================================================== + // TERNARY AND CONDITIONALS + // ===================================================================== + + #[tokio::test] + async fn test_ternary_and_conditionals() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(10)))); + env.insert("y".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("x > y ? 'bigger' : 'smaller'", env.clone(), json!("bigger")).await; + assert_eval("x === 10 ? true : false", env.clone(), json!(true)).await; + assert_eval("x > 5 && y < 10", env.clone(), json!(true)).await; + assert_eval("x > 20 || y < 10", env.clone(), json!(true)).await; + assert_eval("!false", env.clone(), json!(true)).await; + } + + // ===================================================================== + // OBJECT CREATION + // ===================================================================== + + #[tokio::test] + async fn test_object_creation() { + let mut env = HashMap::new(); + env.insert("name".to_string(), Arc::new(to_raw_value(&json!("test")))); + env.insert("value".to_string(), Arc::new(to_raw_value(&json!(42)))); + + assert_eval("({ foo: 'bar' })", env.clone(), json!({"foo": "bar"})).await; + assert_eval( + "({ name, value })", + env.clone(), + json!({"name": "test", "value": 42}), + ) + .await; + assert_eval( + "({ ...{ a: 1 }, b: 2 })", + env.clone(), + json!({"a": 1, "b": 2}), + ) + .await; + } + + // ===================================================================== + // NULL / UNDEFINED + // ===================================================================== + + #[tokio::test] + async fn test_null_undefined() { + assert_eval("null", HashMap::new(), json!(null)).await; + assert_eval("undefined", HashMap::new(), json!(null)).await; + + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(null)))); + assert_eval("x", env.clone(), json!(null)).await; + assert_eval("x ?? 'default'", env.clone(), json!("default")).await; + } + + // ===================================================================== + // FLOW INPUT + // ===================================================================== + + #[tokio::test] + async fn test_flow_input() { + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("test_flow"))); + fi.insert("count".to_string(), to_raw_value(&json!(100))); + fi.insert( + "config".to_string(), + to_raw_value(&json!({"enabled": true})), + ); + + let fi = Some(mappable_rc::Marc::new(fi)); + + assert_eval_full( + "flow_input.name", + HashMap::new(), + fi.clone(), + None, + json!("test_flow"), + ) + .await; + assert_eval_full( + "flow_input.count", + HashMap::new(), + fi.clone(), + None, + json!(100), + ) + .await; + assert_eval_full( + "flow_input.config.enabled", + HashMap::new(), + fi.clone(), + None, + json!(true), + ) + .await; + } + + // ===================================================================== + // TEMPLATE LITERALS + // ===================================================================== + + #[tokio::test] + async fn test_template_literals() { + let mut env = HashMap::new(); + env.insert("name".to_string(), Arc::new(to_raw_value(&json!("World")))); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("`Hello ${name}!`", env.clone(), json!("Hello World!")).await; + assert_eval( + "`The answer is ${x * 2}`", + env.clone(), + json!("The answer is 10"), + ) + .await; + } + + // ===================================================================== + // JSON OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_json_operations() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), + ); + + // JSON.stringify produces a string result + assert_eval( + "JSON.stringify(obj)", + env.clone(), + json!("{\"a\":1,\"b\":2}"), + ) + .await; + assert_eval("Object.keys(obj)", env.clone(), json!(["a", "b"])).await; + assert_eval("Object.values(obj)", env.clone(), json!([1, 2])).await; + } + + // ===================================================================== + // MATH OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_math_operations() { + let env = HashMap::new(); + + assert_eval("Math.max(1, 5, 3)", env.clone(), json!(5)).await; + assert_eval("Math.min(1, 5, 3)", env.clone(), json!(1)).await; + assert_eval("Math.abs(-5)", env.clone(), json!(5)).await; + assert_eval("Math.floor(3.7)", env.clone(), json!(3)).await; + assert_eval("Math.ceil(3.2)", env.clone(), json!(4)).await; + assert_eval("Math.round(3.5)", env.clone(), json!(4)).await; + } + + // ===================================================================== + // TYPE COERCION + // ===================================================================== + + #[tokio::test] + async fn test_type_coercion() { + let mut env = HashMap::new(); + env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); + env.insert("str".to_string(), Arc::new(to_raw_value(&json!("123")))); + + assert_eval("String(num)", env.clone(), json!("42")).await; + assert_eval("Number(str)", env.clone(), json!(123)).await; + assert_eval("Boolean(num)", env.clone(), json!(true)).await; + assert_eval("parseInt('42px')", env.clone(), json!(42)).await; + assert_eval("parseFloat('3.14')", env.clone(), json!(3.14)).await; + } + + // ===================================================================== + // ARRAY SPREAD + // ===================================================================== + + #[tokio::test] + async fn test_array_spread() { + let mut env = HashMap::new(); + env.insert( + "arr1".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3]))), + ); + env.insert( + "arr2".to_string(), + Arc::new(to_raw_value(&json!([4, 5, 6]))), + ); + + assert_eval("[...arr1, ...arr2]", env.clone(), json!([1, 2, 3, 4, 5, 6])).await; + assert_eval("[0, ...arr1, 99]", env.clone(), json!([0, 1, 2, 3, 99])).await; + } + + // ===================================================================== + // MULTILINE STATEMENTS + // ===================================================================== + + #[tokio::test] + async fn test_multiline_statements() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval( + r#"let y = x * 2; + return y + 1"#, + env.clone(), + json!(11), + ) + .await; + + assert_eval( + r#"const result = x > 3 ? 'big' : 'small'; + return result"#, + env.clone(), + json!("big"), + ) + .await; + } + + // ===================================================================== + // OPTIONAL CHAINING & NULLISH COALESCING + // ===================================================================== + + #[tokio::test] + async fn test_optional_chaining_nullish() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"a": {"b": 1}}))), + ); + env.insert("empty".to_string(), Arc::new(to_raw_value(&json!(null)))); + + assert_eval("obj?.a?.b", env.clone(), json!(1)).await; + assert_eval("obj?.a?.c", env.clone(), json!(null)).await; + assert_eval("obj?.x?.y", env.clone(), json!(null)).await; + assert_eval("empty?.foo", env.clone(), json!(null)).await; + + assert_eval("null ?? 'default'", env.clone(), json!("default")).await; + assert_eval("undefined ?? 'default'", env.clone(), json!("default")).await; + assert_eval("0 ?? 'default'", env.clone(), json!(0)).await; + assert_eval("'' ?? 'default'", env.clone(), json!("")).await; + assert_eval("false ?? 'default'", env.clone(), json!(false)).await; + } + + // ===================================================================== + // DESTRUCTURING + // ===================================================================== + + #[tokio::test] + async fn test_destructuring() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"name": "test", "value": 42}))), + ); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), + ); + + assert_eval( + "const { name, value } = obj; return { name, value }", + env.clone(), + json!({"name": "test", "value": 42}), + ) + .await; + + assert_eval( + "const [first, second, ...rest] = arr; return { first, second, rest }", + env.clone(), + json!({"first": 1, "second": 2, "rest": [3, 4, 5]}), + ) + .await; + + assert_eval( + "const { missing = 'default' } = obj; return missing", + env.clone(), + json!("default"), + ) + .await; + } + + // ===================================================================== + // NUMBER EDGE CASES + // ===================================================================== + + #[tokio::test] + async fn test_number_edge_cases() { + let env = HashMap::new(); + + assert_eval( + "Number.MAX_SAFE_INTEGER", + env.clone(), + json!(9007199254740991_i64), + ) + .await; + assert_eval( + "Number.MIN_SAFE_INTEGER", + env.clone(), + json!(-9007199254740991_i64), + ) + .await; + assert_eval("Number.isInteger(5)", env.clone(), json!(true)).await; + assert_eval("Number.isInteger(5.5)", env.clone(), json!(false)).await; + assert_eval("Number.isFinite(Infinity)", env.clone(), json!(false)).await; + assert_eval("Number.isNaN(NaN)", env.clone(), json!(true)).await; + assert_eval("isNaN(NaN)", env.clone(), json!(true)).await; + assert_eval("isFinite(100)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // REGEX BASIC + // ===================================================================== + + #[tokio::test] + async fn test_regex_basic() { + let mut env = HashMap::new(); + env.insert( + "str".to_string(), + Arc::new(to_raw_value(&json!("hello world 123"))), + ); + + assert_eval("/hello/.test(str)", env.clone(), json!(true)).await; + assert_eval("str.match(/\\d+/)?.[0]", env.clone(), json!("123")).await; + assert_eval( + "str.replace(/world/, 'universe')", + env.clone(), + json!("hello universe 123"), + ) + .await; + assert_eval( + "str.split(/\\s+/)", + env.clone(), + json!(["hello", "world", "123"]), + ) + .await; + assert_eval("'aaa'.replace(/a/g, 'b')", env.clone(), json!("bbb")).await; + assert_eval("/HELLO/i.test(str)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // DATE OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_date_basic() { + let env = HashMap::new(); + + assert_eval( + "Date.parse('2024-01-15T00:00:00.000Z')", + env.clone(), + json!(1705276800000_i64), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCFullYear()", + env.clone(), + json!(2024), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCMonth()", + env.clone(), + json!(0), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCDate()", + env.clone(), + json!(15), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').toISOString()", + env.clone(), + json!("2024-01-15T00:00:00.000Z"), + ) + .await; + } + + #[tokio::test] + async fn test_date_serialization() { + let env = HashMap::new(); + + // Direct Date → ISO string via toJSON + assert_eval( + "new Date('2024-01-15T12:30:00.000Z')", + env.clone(), + json!("2024-01-15T12:30:00.000Z"), + ) + .await; + + // Date within an object + assert_eval( + "({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })", + env.clone(), + json!({"date": "2024-01-15T00:00:00.000Z", "name": "test"}), + ) + .await; + + // Deeply nested Date + assert_eval( + "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })", + env.clone(), + json!({"level1": {"level2": {"date": "2024-01-15T00:00:00.000Z"}}}), + ) + .await; + } + + // ===================================================================== + // SPECIAL OBJECT SERIALIZATION + // ===================================================================== + + #[tokio::test] + async fn test_special_object_serialization() { + let env = HashMap::new(); + + // RegExp, Map, Set all serialize to {} + assert_eval("/test/gi", env.clone(), json!({})).await; + assert_eval("new Map([['key', 'value']])", env.clone(), json!({})).await; + assert_eval("new Set([1, 2, 3])", env.clone(), json!({})).await; + } + + // ===================================================================== + // ARRAY ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_array_advanced() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9, 2, 6]))), + ); + env.insert( + "nested".to_string(), + Arc::new(to_raw_value(&json!([[1, 2], [3, 4], [5, 6]]))), + ); + + assert_eval( + "[...arr].sort((a, b) => a - b)", + env.clone(), + json!([1, 1, 2, 3, 4, 5, 6, 9]), + ) + .await; + assert_eval( + "[...arr].sort((a, b) => b - a)", + env.clone(), + json!([9, 6, 5, 4, 3, 2, 1, 1]), + ) + .await; + assert_eval("nested.flat()", env.clone(), json!([1, 2, 3, 4, 5, 6])).await; + assert_eval( + "nested.flatMap(x => x)", + env.clone(), + json!([1, 2, 3, 4, 5, 6]), + ) + .await; + assert_eval("arr.indexOf(5)", env.clone(), json!(4)).await; + assert_eval("arr.indexOf(99)", env.clone(), json!(-1)).await; + assert_eval("arr.includes(9)", env.clone(), json!(true)).await; + assert_eval("arr.slice(2, 5)", env.clone(), json!([4, 1, 5])).await; + assert_eval("arr.slice(-3)", env.clone(), json!([9, 2, 6])).await; + } + + // ===================================================================== + // LOGICAL OPERATORS + // ===================================================================== + + #[tokio::test] + async fn test_logical_operators() { + let mut env = HashMap::new(); + env.insert("a".to_string(), Arc::new(to_raw_value(&json!(true)))); + env.insert("b".to_string(), Arc::new(to_raw_value(&json!(false)))); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("a && 'yes'", env.clone(), json!("yes")).await; + assert_eval("b && 'yes'", env.clone(), json!(false)).await; + assert_eval("b || 'no'", env.clone(), json!("no")).await; + assert_eval("a || 'no'", env.clone(), json!(true)).await; + assert_eval("let y = null; y ??= 10; return y", env.clone(), json!(10)).await; + assert_eval("let y = 5; y ??= 10; return y", env.clone(), json!(5)).await; + assert_eval("(a && x > 3) || (b && x < 3)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // TYPEOF + // ===================================================================== + + #[tokio::test] + async fn test_typeof() { + let mut env = HashMap::new(); + env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); + env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); + env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([1, 2, 3])))); + env.insert("obj".to_string(), Arc::new(to_raw_value(&json!({"a": 1})))); + env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); + + assert_eval("typeof str", env.clone(), json!("string")).await; + assert_eval("typeof num", env.clone(), json!("number")).await; + assert_eval("typeof arr", env.clone(), json!("object")).await; + assert_eval("typeof obj", env.clone(), json!("object")).await; + assert_eval("typeof n", env.clone(), json!("object")).await; + assert_eval("typeof undefined", env.clone(), json!("undefined")).await; + assert_eval("Array.isArray(arr)", env.clone(), json!(true)).await; + assert_eval("Array.isArray(obj)", env.clone(), json!(false)).await; + } + + // ===================================================================== + // COMPLEX MULTILINE EXPRESSIONS + // ===================================================================== + + #[tokio::test] + async fn test_multiline_complex_logic() { + let mut env = HashMap::new(); + env.insert( + "users".to_string(), + Arc::new(to_raw_value(&json!([ + {"name": "Alice", "age": 30, "role": "admin"}, + {"name": "Bob", "age": 25, "role": "user"}, + {"name": "Charlie", "age": 35, "role": "admin"}, + {"name": "Diana", "age": 28, "role": "user"} + ]))), + ); + + assert_eval( + r#" + const admins = users.filter(u => u.role === 'admin'); + const names = admins.map(u => u.name); + return names.join(', ') + "#, + env.clone(), + json!("Alice, Charlie"), + ) + .await; + + assert_eval( + r#" + const totalAge = users.reduce((sum, u) => sum + u.age, 0); + const avgAge = totalAge / users.length; + return Math.round(avgAge) + "#, + env.clone(), + json!(30), + ) + .await; + + assert_eval( + r#" + const grouped = users.reduce((acc, u) => { + if (!acc[u.role]) acc[u.role] = []; + acc[u.role].push(u.name); + return acc; + }, {}); + return grouped + "#, + env.clone(), + json!({"admin": ["Alice", "Charlie"], "user": ["Bob", "Diana"]}), + ) + .await; + } + + // ===================================================================== + // DATA TRANSFORMATION + // ===================================================================== + + #[tokio::test] + async fn test_multiline_data_transformation() { + let mut env = HashMap::new(); + env.insert( + "data".to_string(), + Arc::new(to_raw_value(&json!({ + "items": [ + {"id": 1, "price": 100, "quantity": 2}, + {"id": 2, "price": 50, "quantity": 5}, + {"id": 3, "price": 75, "quantity": 3} + ], + "discount": 0.1 + }))), + ); + + assert_eval( + r#" + const subtotals = data.items.map(item => item.price * item.quantity); + const total = subtotals.reduce((a, b) => a + b, 0); + const discounted = total * (1 - data.discount); + return { subtotals, total, discounted } + "#, + env.clone(), + json!({"subtotals": [200, 250, 225], "total": 675, "discounted": 607.5}), + ) + .await; + } + + // ===================================================================== + // TRY-CATCH + // ===================================================================== + + #[tokio::test] + async fn test_try_catch() { + let env = HashMap::new(); + + assert_eval( + r#" + try { + return JSON.parse('{"valid": true}'); + } catch (e) { + return { problem: e.message }; + } + "#, + env.clone(), + json!({"valid": true}), + ) + .await; + + assert_eval( + r#" + try { + return JSON.parse('invalid json'); + } catch (e) { + return { problem: 'parse_failed' }; + } + "#, + env.clone(), + json!({"problem": "parse_failed"}), + ) + .await; + + assert_eval( + r#" + let result = 'initial'; + try { + result = 'try'; + } catch (e) { + result = 'catch'; + } finally { + result = result + '_finally'; + } + return result + "#, + env.clone(), + json!("try_finally"), + ) + .await; + } + + // ===================================================================== + // OBJECT ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_object_advanced() { + let mut env = HashMap::new(); + env.insert( + "config".to_string(), + Arc::new(to_raw_value(&json!({ + "server": {"host": "localhost", "port": 8080}, + "database": {"host": "db.local", "port": 5432}, + "features": ["auth", "logging", "cache"] + }))), + ); + + assert_eval( + "Object.assign({}, config.server, { secure: true })", + env.clone(), + json!({"host": "localhost", "port": 8080, "secure": true}), + ) + .await; + + assert_eval( + "({ ...config.server, port: 443, secure: true })", + env.clone(), + json!({"host": "localhost", "port": 443, "secure": true}), + ) + .await; + + assert_eval( + "JSON.parse(JSON.stringify(config))", + env.clone(), + json!({ + "server": {"host": "localhost", "port": 8080}, + "database": {"host": "db.local", "port": 5432}, + "features": ["auth", "logging", "cache"] + }), + ) + .await; + + assert_eval( + r#" + const key = 'dynamic'; + return { [key]: 'value', [`${key}_2`]: 'value2' } + "#, + env.clone(), + json!({"dynamic": "value", "dynamic_2": "value2"}), + ) + .await; + } + + // ===================================================================== + // STRING ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_string_advanced() { + let mut env = HashMap::new(); + env.insert( + "text".to_string(), + Arc::new(to_raw_value(&json!(" Hello, World! "))), + ); + env.insert( + "path".to_string(), + Arc::new(to_raw_value(&json!("/api/v1/users/123/profile"))), + ); + + assert_eval("text.trim()", env.clone(), json!("Hello, World!")).await; + assert_eval("text.trimStart()", env.clone(), json!("Hello, World! ")).await; + assert_eval("text.trimEnd()", env.clone(), json!(" Hello, World!")).await; + assert_eval("'42'.padStart(5, '0')", env.clone(), json!("00042")).await; + assert_eval("'42'.padEnd(5, '-')", env.clone(), json!("42---")).await; + assert_eval("'ab'.repeat(3)", env.clone(), json!("ababab")).await; + assert_eval( + "path.split('/').filter(p => p.length > 0)", + env.clone(), + json!(["api", "v1", "users", "123", "profile"]), + ) + .await; + } + + // ===================================================================== + // ARRAY MANIPULATION + // ===================================================================== + + #[tokio::test] + async fn test_array_manipulation() { + let mut env = HashMap::new(); + env.insert( + "items".to_string(), + Arc::new(to_raw_value(&json!([ + {"id": 1, "name": "Apple", "category": "fruit"}, + {"id": 2, "name": "Carrot", "category": "vegetable"}, + {"id": 3, "name": "Banana", "category": "fruit"}, + {"id": 4, "name": "Broccoli", "category": "vegetable"} + ]))), + ); + + assert_eval( + "items.find(i => i.name === 'Banana')", + env.clone(), + json!({"id": 3, "name": "Banana", "category": "fruit"}), + ) + .await; + assert_eval( + "items.findIndex(i => i.name === 'Banana')", + env.clone(), + json!(2), + ) + .await; + assert_eval( + "items.filter(i => i.category === 'fruit').map(i => i.name).sort()", + env.clone(), + json!(["Apple", "Banana"]), + ) + .await; + assert_eval( + "Array.from({length: 5}, (_, i) => i * 2)", + env.clone(), + json!([0, 2, 4, 6, 8]), + ) + .await; + assert_eval("Array(3).fill(0)", env.clone(), json!([0, 0, 0])).await; + assert_eval( + "[1, 2].concat([3, 4], [5, 6])", + env.clone(), + json!([1, 2, 3, 4, 5, 6]), + ) + .await; + assert_eval( + "items.map(i => i.name).join(' | ')", + env.clone(), + json!("Apple | Carrot | Banana | Broccoli"), + ) + .await; + } + + // ===================================================================== + // SET AND MAP OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_set_operations() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3, 4]))), + ); + + assert_eval("[...new Set(arr)]", env.clone(), json!([1, 2, 3, 4])).await; + assert_eval("new Set(arr).size", env.clone(), json!(4)).await; + assert_eval("new Set(arr).has(3)", env.clone(), json!(true)).await; + assert_eval("new Set(arr).has(99)", env.clone(), json!(false)).await; + } + + #[tokio::test] + async fn test_map_operations() { + let env = HashMap::new(); + + assert_eval( + r#" + const map = new Map([['a', 1], ['b', 2], ['c', 3]]); + return Object.fromEntries(map) + "#, + env.clone(), + json!({"a": 1, "b": 2, "c": 3}), + ) + .await; + + assert_eval( + r#" + const map = new Map(); + map.set('key1', 'value1'); + map.set('key2', 'value2'); + return map.get('key1') + "#, + env.clone(), + json!("value1"), + ) + .await; + + assert_eval( + r#" + const map = new Map([['a', 1], ['b', 2]]); + return map.size + "#, + env.clone(), + json!(2), + ) + .await; + } + + // ===================================================================== + // COMPARISONS + // ===================================================================== + + #[tokio::test] + async fn test_comparisons() { + let env = HashMap::new(); + + assert_eval("1 === 1", env.clone(), json!(true)).await; + assert_eval("1 === '1'", env.clone(), json!(false)).await; + assert_eval("null === undefined", env.clone(), json!(false)).await; + assert_eval("null === null", env.clone(), json!(true)).await; + assert_eval("1 == '1'", env.clone(), json!(true)).await; + assert_eval("null == undefined", env.clone(), json!(true)).await; + assert_eval("5 !== '5'", env.clone(), json!(true)).await; + assert_eval("5 > 3", env.clone(), json!(true)).await; + assert_eval("5 >= 5", env.clone(), json!(true)).await; + assert_eval("3 < 5", env.clone(), json!(true)).await; + assert_eval("5 <= 5", env.clone(), json!(true)).await; + assert_eval("'apple' < 'banana'", env.clone(), json!(true)).await; + } + + // ===================================================================== + // BITWISE OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_bitwise() { + let env = HashMap::new(); + + assert_eval("5 & 3", env.clone(), json!(1)).await; + assert_eval("5 | 3", env.clone(), json!(7)).await; + assert_eval("5 ^ 3", env.clone(), json!(6)).await; + assert_eval("~5", env.clone(), json!(-6)).await; + assert_eval("5 << 2", env.clone(), json!(20)).await; + assert_eval("20 >> 2", env.clone(), json!(5)).await; + } + + // ===================================================================== + // REAL-WORLD FLOW PATTERNS + // ===================================================================== + + #[tokio::test] + async fn test_flow_patterns_api_response() { + let mut env = HashMap::new(); + env.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({ + "status": 200, + "data": { + "users": [ + {"id": 1, "email": "alice@example.com", "active": true}, + {"id": 2, "email": "bob@example.com", "active": false}, + {"id": 3, "email": "charlie@example.com", "active": true} + ], + "pagination": {"page": 1, "total": 50, "per_page": 10} + } + }))), + ); + + assert_eval( + "previous_result.data.users.filter(u => u.active).map(u => u.email)", + env.clone(), + json!(["alice@example.com", "charlie@example.com"]), + ) + .await; + + assert_eval( + r#" + const { page, total, per_page } = previous_result.data.pagination; + return page * per_page < total + "#, + env.clone(), + json!(true), + ) + .await; + } + + #[tokio::test] + async fn test_flow_patterns_batch_processing() { + let mut env = HashMap::new(); + env.insert( + "jobs".to_string(), + Arc::new(to_raw_value(&json!([ + {"id": 1, "status": "completed", "result": 100}, + {"id": 2, "status": "failed", "reason": "timeout"}, + {"id": 3, "status": "completed", "result": 200}, + {"id": 4, "status": "failed", "reason": "connection"}, + {"id": 5, "status": "completed", "result": 150} + ]))), + ); + + assert_eval( + r#" + const completed = jobs.filter(j => j.status === 'completed'); + const failed = jobs.filter(j => j.status === 'failed'); + const totalResult = completed.reduce((sum, j) => sum + j.result, 0); + return { + totalJobs: jobs.length, + completedCount: completed.length, + failedCount: failed.length, + successRate: completed.length / jobs.length, + totalResult, + failureReasons: failed.map(j => j.reason) + } + "#, + env.clone(), + json!({ + "totalJobs": 5, + "completedCount": 3, + "failedCount": 2, + "successRate": 0.6, + "totalResult": 450, + "failureReasons": ["timeout", "connection"] + }), + ) + .await; + } + + #[tokio::test] + async fn test_flow_patterns_config_merge() { + let mut env = HashMap::new(); + env.insert( + "defaults".to_string(), + Arc::new(to_raw_value(&json!({ + "timeout": 5000, + "retries": 3, + "headers": {"Content-Type": "application/json"}, + "features": {"logging": true, "caching": false} + }))), + ); + env.insert( + "overrides".to_string(), + Arc::new(to_raw_value(&json!({ + "timeout": 10000, + "headers": {"Authorization": "Bearer token"}, + "features": {"caching": true} + }))), + ); + + assert_eval( + r#"({ + ...defaults, + ...overrides, + headers: { ...defaults.headers, ...overrides.headers }, + features: { ...defaults.features, ...overrides.features } + })"#, + env.clone(), + json!({ + "timeout": 10000, + "retries": 3, + "headers": {"Content-Type": "application/json", "Authorization": "Bearer token"}, + "features": {"logging": true, "caching": true} + }), + ) + .await; + } + + // ===================================================================== + // FLOW SIMULATION: Multi-step flow with various step results + // ===================================================================== + + fn create_multi_step_flow_context() -> ( + HashMap>>, + Option>>>, + HashMap>, + ) { + let mut ctx = HashMap::new(); + ctx.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); + ctx.insert( + "b".to_string(), + Arc::new(to_raw_value(&json!({ + "status": "success", + "data": { + "users": [ + {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, + {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, + {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} + ], + "total": 3, + "metadata": {"page": 1, "hasMore": true} + } + }))), + ); + ctx.insert( + "c".to_string(), + Arc::new(to_raw_value(&json!([10, 20, 30, 40, 50]))), + ); + ctx.insert("d".to_string(), Arc::new(to_raw_value(&json!(null)))); + ctx.insert( + "f".to_string(), + Arc::new(to_raw_value(&json!({ + "level1": {"level2": {"level3": {"level4": {"value": "deeply_nested"}}}} + }))), + ); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!([ + "string", 123, true, null, {"key": "value"}, [1, 2, 3] + ]))), + ); + + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("test_flow"))); + fi.insert("count".to_string(), to_raw_value(&json!(100))); + fi.insert("enabled".to_string(), to_raw_value(&json!(true))); + fi.insert( + "config".to_string(), + to_raw_value(&json!({ + "timeout": 30, "retries": 3, "options": ["fast", "secure"] + })), + ); + fi.insert( + "items".to_string(), + to_raw_value(&json!([ + {"id": 1, "value": "first"}, + {"id": 2, "value": "second"}, + {"id": 3, "value": "third"} + ])), + ); + + let mut fe = HashMap::new(); + fe.insert("ENV".to_string(), to_raw_value(&json!("production"))); + fe.insert("DEBUG".to_string(), to_raw_value(&json!(false))); + fe.insert("VERSION".to_string(), to_raw_value(&json!("1.2.3"))); + + (ctx, Some(mappable_rc::Marc::new(fi)), fe) + } + + #[tokio::test] + async fn test_flow_step_references() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full("a", ctx.clone(), fi.clone(), Some(&fe), json!(42)).await; + assert_eval_full( + "b.status", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("success"), + ) + .await; + assert_eval_full("b.data.total", ctx.clone(), fi.clone(), Some(&fe), json!(3)).await; + assert_eval_full( + "b.data.users[0].name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("Alice"), + ) + .await; + assert_eval_full("c[0]", ctx.clone(), fi.clone(), Some(&fe), json!(10)).await; + assert_eval_full( + "c.map(x => x * 2)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!([20, 40, 60, 80, 100]), + ) + .await; + assert_eval_full( + "f.level1.level2.level3.level4.value", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("deeply_nested"), + ) + .await; + } + + #[tokio::test] + async fn test_flow_input_nested_and_combined() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "flow_input.name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("test_flow"), + ) + .await; + assert_eval_full( + "flow_input.config.timeout", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(30), + ) + .await; + assert_eval_full( + "flow_input.config.options[0]", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("fast"), + ) + .await; + assert_eval_full( + "flow_input.items.length", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(3), + ) + .await; + assert_eval_full( + "flow_input.items[0].id", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(1), + ) + .await; + assert_eval_full( + "flow_input.items.map(i => i.value)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(["first", "second", "third"]), + ) + .await; + + // Combining flow_input with step results + assert_eval_full( + "flow_input.count + a", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(142), + ) + .await; + assert_eval_full( + "flow_input.enabled ? b.data.users : []", + ctx.clone(), + fi.clone(), + Some(&fe), + json!([ + {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, + {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, + {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} + ]), + ) + .await; + } + + #[tokio::test] + async fn test_flow_env_access() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "flow_env.ENV", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("production"), + ) + .await; + assert_eval_full( + "flow_env.DEBUG", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(false), + ) + .await; + assert_eval_full( + "flow_env.VERSION", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("1.2.3"), + ) + .await; + } + + #[tokio::test] + async fn test_flow_branch_conditions() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full("a > 40", ctx.clone(), fi.clone(), Some(&fe), json!(true)).await; + assert_eval_full( + "b.status === 'success'", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "flow_input.enabled", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "a > 40 && b.status === 'success'", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "b.data.users.length > 0", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "b.data.users.some(u => u.active)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "c.includes(30)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + } + + #[tokio::test] + async fn test_flow_complex_data_extraction() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "b.data.users.filter(u => u.active).map(u => u.name)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(["Alice", "Charlie"]), + ) + .await; + + assert_eval_full( + "b.data.users.filter(u => u.roles.includes('admin'))[0]?.name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("Alice"), + ) + .await; + + assert_eval_full( + "b.data.users.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(2), + ) + .await; + + // Combining multiple step results + assert_eval_full("a + c[0]", ctx.clone(), fi.clone(), Some(&fe), json!(52)).await; + assert_eval_full( + "a * b.data.total", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(126), + ) + .await; + } + + #[tokio::test] + async fn test_flow_forloop_inner_expressions() { + let mut ctx = HashMap::new(); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({"value": 42, "index": 2}))), + ); + + let mut fi = HashMap::new(); + fi.insert( + "iter".to_string(), + to_raw_value(&json!({"index": 2, "value": {"id": 3, "name": "test_item"}})), + ); + fi.insert("name".to_string(), to_raw_value(&json!("parent_flow"))); + let fi = Some(mappable_rc::Marc::new(fi)); + + assert_eval_full( + "flow_input.iter.index", + ctx.clone(), + fi.clone(), + None, + json!(2), + ) + .await; + assert_eval_full( + "flow_input.iter.value.id", + ctx.clone(), + fi.clone(), + None, + json!(3), + ) + .await; + assert_eval_full( + "flow_input.iter.value.name", + ctx.clone(), + fi.clone(), + None, + json!("test_item"), + ) + .await; + assert_eval_full( + "`Item ${flow_input.iter.index} of ${flow_input.name}`", + ctx.clone(), + fi.clone(), + None, + json!("Item 2 of parent_flow"), + ) + .await; + } + + // ===================================================================== + // EDGE CASES + // ===================================================================== + + #[tokio::test] + async fn test_edge_cases_empty_values() { + let mut env = HashMap::new(); + env.insert("emptyArray".to_string(), Arc::new(to_raw_value(&json!([])))); + env.insert( + "emptyObject".to_string(), + Arc::new(to_raw_value(&json!({}))), + ); + env.insert( + "emptyString".to_string(), + Arc::new(to_raw_value(&json!(""))), + ); + env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); + + assert_eval("emptyArray.length", env.clone(), json!(0)).await; + assert_eval("emptyArray.map(x => x * 2)", env.clone(), json!([])).await; + assert_eval( + "emptyArray.reduce((a, b) => a + b, 100)", + env.clone(), + json!(100), + ) + .await; + assert_eval("Object.keys(emptyObject)", env.clone(), json!([])).await; + assert_eval("emptyString.length", env.clone(), json!(0)).await; + assert_eval("emptyString || 'default'", env.clone(), json!("default")).await; + assert_eval("emptyString ?? 'default'", env.clone(), json!("")).await; + assert_eval("zero || 'default'", env.clone(), json!("default")).await; + assert_eval("zero ?? 'default'", env.clone(), json!(0)).await; + } + + #[tokio::test] + async fn test_edge_cases_deep_access() { + let mut env = HashMap::new(); + env.insert( + "deep".to_string(), + Arc::new(to_raw_value( + &json!({"a": {"b": {"c": {"d": {"e": "found!"}}}}}), + )), + ); + + assert_eval("deep.a.b.c.d.e", env.clone(), json!("found!")).await; + assert_eval("deep?.a?.b?.c?.d?.e", env.clone(), json!("found!")).await; + assert_eval("deep?.a?.b?.x?.y?.z", env.clone(), json!(null)).await; + assert_eval( + "deep?.a?.b?.x?.y?.z ?? 'not found'", + env.clone(), + json!("not found"), + ) + .await; + } + + #[tokio::test] + async fn test_edge_cases_special_characters() { + let mut env = HashMap::new(); + env.insert( + "data".to_string(), + Arc::new(to_raw_value(&json!({ + "key-with-dash": "value1", + "key.with.dots": "value2", + "key with spaces": "value3" + }))), + ); + + assert_eval("data['key-with-dash']", env.clone(), json!("value1")).await; + assert_eval("data['key.with.dots']", env.clone(), json!("value2")).await; + assert_eval("data['key with spaces']", env.clone(), json!("value3")).await; + } + + #[tokio::test] + async fn test_edge_cases_boolean_coercion() { + let env = HashMap::new(); + + assert_eval("Boolean(0)", env.clone(), json!(false)).await; + assert_eval("Boolean('')", env.clone(), json!(false)).await; + assert_eval("Boolean(null)", env.clone(), json!(false)).await; + assert_eval("Boolean(undefined)", env.clone(), json!(false)).await; + assert_eval("Boolean(NaN)", env.clone(), json!(false)).await; + assert_eval("Boolean(1)", env.clone(), json!(true)).await; + assert_eval("Boolean('hello')", env.clone(), json!(true)).await; + assert_eval("Boolean([])", env.clone(), json!(true)).await; + assert_eval("Boolean({})", env.clone(), json!(true)).await; + assert_eval("!!0", env.clone(), json!(false)).await; + assert_eval("!!1", env.clone(), json!(true)).await; + } + + // ===================================================================== + // PROMISE RESOLUTION + // ===================================================================== + + #[tokio::test] + async fn test_promise_resolve() { + let env = HashMap::new(); + + assert_eval("Promise.resolve(42)", env.clone(), json!(42)).await; + assert_eval( + "Promise.resolve({ key: 'value' })", + env.clone(), + json!({"key": "value"}), + ) + .await; + assert_eval( + "Promise.all([Promise.resolve(1), Promise.resolve(2), Promise.resolve(3)])", + env.clone(), + json!([1, 2, 3]), + ) + .await; + } + + // ===================================================================== + // eval_simple_js TESTS + // ===================================================================== + + #[tokio::test] + async fn test_eval_simple_js_basic() { + let mut globals = HashMap::new(); + globals.insert("job".to_string(), json!({"id": "abc", "input": {"x": 42}})); + + let result = eval_simple_js("job.input.x".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "42"); + } + + #[tokio::test] + async fn test_eval_simple_js_string() { + let mut globals = HashMap::new(); + globals.insert("name".to_string(), json!("World")); + + let result = eval_simple_js("`Hello ${name}!`".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "\"Hello World!\""); + } + + #[tokio::test] + async fn test_eval_simple_js_array_transform() { + let mut globals = HashMap::new(); + globals.insert("data".to_string(), json!([1, 2, 3, 4, 5])); + + let result = eval_simple_js( + "data.filter(x => x > 2).map(x => x * 10)".to_string(), + globals, + ) + .await + .unwrap(); + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap(); + assert_eq!(actual, json!([30, 40, 50])); + } + + #[tokio::test] + async fn test_eval_simple_js_null_handling() { + let mut globals = HashMap::new(); + globals.insert("x".to_string(), json!(null)); + + let result = eval_simple_js("x ?? 'default'".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "\"default\""); + } + + #[tokio::test] + async fn test_eval_simple_js_multiple_globals() { + let mut globals = HashMap::new(); + globals.insert("a".to_string(), json!(10)); + globals.insert("b".to_string(), json!(20)); + globals.insert("prefix".to_string(), json!("result")); + + let result = eval_simple_js( + "({ label: `${prefix}_sum`, value: a + b })".to_string(), + globals, + ) + .await + .unwrap(); + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap(); + assert_eq!(actual, json!({"label": "result_sum", "value": 30})); + } + + #[tokio::test] + async fn test_eval_simple_js_error_handling() { + let globals = HashMap::new(); + + // Invalid JS should return an error + let result = eval_simple_js("this is not valid js @#$".to_string(), globals).await; + assert!(result.is_err()); + } +} diff --git a/backend/windmill-runtime-nativets/Cargo.toml b/backend/windmill-runtime-nativets/Cargo.toml new file mode 100644 index 0000000000..cb084016a8 --- /dev/null +++ b/backend/windmill-runtime-nativets/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "windmill-runtime-nativets" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_runtime_nativets" +path = "src/lib.rs" + +[features] +default = [] +private = [] +enterprise = ["windmill-common/enterprise"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-parser-ts.workspace = true +deno_fetch.workspace = true +deno_webidl.workspace = true +deno_web.workspace = true +deno_net.workspace = true +deno_console.workspace = true +deno_url.workspace = true +deno_core.workspace = true +deno_ast.workspace = true +deno_tls.workspace = true +deno_permissions.workspace = true +deno_io.workspace = true +deno_telemetry.workspace = true +deno_error.workspace = true +deno_runtime.workspace = true +winapi.workspace = true + +itertools.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +anyhow.workspace = true +uuid.workspace = true +reqwest.workspace = true +regex.workspace = true +lazy_static.workspace = true +const_format.workspace = true +futures.workspace = true +sqlx.workspace = true + +[build-dependencies] +deno_fetch.workspace = true +deno_webidl.workspace = true +deno_web.workspace = true +deno_net.workspace = true +deno_console.workspace = true +deno_url.workspace = true +deno_core.workspace = true +deno_ast.workspace = true +deno_tls.workspace = true +deno_permissions.workspace = true +deno_io.workspace = true +deno_runtime.workspace = true +deno_telemetry.workspace = true +winapi.workspace = true diff --git a/backend/windmill-runtime-nativets/build.rs b/backend/windmill-runtime-nativets/build.rs new file mode 100644 index 0000000000..2f4ec8c38d --- /dev/null +++ b/backend/windmill-runtime-nativets/build.rs @@ -0,0 +1,124 @@ +use deno_fetch::FetchPermissions; +use deno_net::NetPermissions; +use deno_web::{BlobStore, TimersPermission}; +use std::borrow::Cow; +use std::env; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub struct PermissionsContainer; + +impl FetchPermissions for PermissionsContainer { + #[inline(always)] + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + unreachable!("snapshotting") + } + + #[inline(always)] + fn check_read<'a>( + &mut self, + _resolved: bool, + _p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_io::fs::FsError> { + unreachable!("snapshotting") + } +} + +impl TimersPermission for PermissionsContainer { + #[inline(always)] + fn allow_hrtime(&mut self) -> bool { + true + } +} + +impl NetPermissions for PermissionsContainer { + fn check_read<'a>( + &mut self, + _p: &'a str, + _api_name: &str, + ) -> Result { + unreachable!("snapshotting") + } + + fn check_write<'a>( + &mut self, + _p: &'a str, + _api_name: &str, + ) -> Result { + unreachable!("snapshotting") + } + + fn check_net>( + &mut self, + _host: &(T, Option), + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + unreachable!("snapshotting") + } + + fn check_write_path<'a>( + &mut self, + _: &'a Path, + _: &str, + ) -> Result, deno_permissions::PermissionCheckError> { + todo!() + } +} + +deno_core::extension!( + fetch, + esm_entry_point = "ext:fetch/src/runtime.js", + esm = ["src/runtime.js"], +); + +fn main() { + println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); + println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + + let exts = vec![ + deno_telemetry::deno_telemetry::init_ops_and_esm(), + deno_webidl::deno_webidl::init_ops_and_esm(), + deno_url::deno_url::init_ops_and_esm(), + deno_console::deno_console::init_ops_and_esm(), + deno_web::deno_web::init_ops_and_esm::( + Arc::new(BlobStore::default()), + None, + ), + deno_fetch::deno_fetch::init_ops_and_esm::(Default::default()), + deno_net::deno_net::init_ops_and_esm::(None, None), + fetch::init_ops_and_esm(), + ]; + + // Build the file path to the snapshot. + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); + + // Create the snapshot. + let output = deno_core::snapshot::create_snapshot( + deno_core::snapshot::CreateSnapshotOptions { + cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), + startup_snapshot: None, + extension_transpiler: Some(std::rc::Rc::new(|specifier, source| { + deno_runtime::transpile::maybe_transpile_source(specifier, source) + })), + extensions: exts, + with_runtime_cb: None, + skip_op_registration: false, + }, + None, + ) + .unwrap(); + + let mut file = std::fs::File::create(snapshot_path).unwrap(); + file.write_all(&output.output).unwrap(); + + for path in output.files_loaded_during_snapshot { + println!("cargo:rerun-if-changed={}", path.display()); + } +} diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs new file mode 100644 index 0000000000..57f10fd327 --- /dev/null +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -0,0 +1,710 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Isolated deno_core runtime for NativeTS script execution. +//! +//! This crate encapsulates all deno_core/V8 dependencies for executing +//! TypeScript scripts via the nativets runtime. By isolating this here, +//! deno_core compilation no longer blocks windmill-worker or windmill-api. + +use std::{borrow::Cow, cell::RefCell, path::PathBuf, rc::Rc, sync::Arc}; + +// Re-export deno_telemetry for use by windmill-worker's otel proxy +pub use deno_telemetry; + +use deno_ast::ParseParams; +use deno_core::{ + op2, serde_v8, url, + v8::{self, IsolateHandle}, + Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, +}; +use deno_fetch::FetchPermissions; +use deno_net::NetPermissions; +use deno_web::{BlobStore, TimersPermission}; +use itertools::Itertools; +use lazy_static::lazy_static; +use regex::Regex; +use serde_json::value::RawValue; +use sqlx::types::Json; +use tokio::sync::{mpsc, oneshot}; +use uuid::Uuid; + +use windmill_common::error::Error; +use windmill_common::result_stream::append_result_stream_db; +use windmill_common::worker::{write_file, Connection, TMP_DIR}; + +// ── Permission container ───────────────────────────────────────────── + +pub struct PermissionsContainer; + +impl FetchPermissions for PermissionsContainer { + #[inline(always)] + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + Ok(()) + } + + #[inline(always)] + fn check_read<'a>( + &mut self, + _resolved: bool, + p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_io::fs::FsError> { + Ok(Cow::Borrowed(p)) + } +} + +impl TimersPermission for PermissionsContainer { + #[inline(always)] + fn allow_hrtime(&mut self) -> bool { + true + } +} + +impl NetPermissions for PermissionsContainer { + fn check_read<'a>( + &mut self, + p: &'a str, + _api_name: &str, + ) -> Result { + Ok(PathBuf::from(p)) + } + + fn check_write<'a>( + &mut self, + p: &'a str, + _api_name: &str, + ) -> Result { + Ok(PathBuf::from(p)) + } + + fn check_net>( + &mut self, + _host: &(T, Option), + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + Ok(()) + } + + fn check_write_path<'a>( + &mut self, + p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_permissions::PermissionCheckError> { + Ok(Cow::Borrowed(p)) + } +} + +// ── Types ──────────────────────────────────────────────────────────── + +struct MainArgs { + args: Vec>>, +} + +struct LogString { + pub s: mpsc::UnboundedSender, +} + +pub struct NativeAnnotation { + pub useragent: Option, + pub proxy: Option<(String, Option<(String, String)>)>, +} + +// ── Statics ────────────────────────────────────────────────────────── + +static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); + +const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); + +const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); + +lazy_static! { + static ref RE_PROXY: Regex = + Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); +} + +// ── Public interface ───────────────────────────────────────────────── + +/// Set up the deno_core/V8 runtime. Must be called once before creating any JsRuntime. +pub fn setup_deno_runtime() -> anyhow::Result<()> { + let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ + "--stack-size=1024".to_string(), + "--no-harmony-import-assertions".to_string(), + ]) + .into_iter() + .skip(1) + .collect::>(); + + if !unrecognized_v8_flags.is_empty() { + println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags); + } + + deno_core::JsRuntime::init_platform(None, false); + Ok(()) +} + +pub fn transpile_ts(expr: String) -> anyhow::Result { + let parsed = deno_ast::parse_module(ParseParams { + specifier: url::Url::parse("file:///eval.ts")?, + capture_tokens: false, + scope_analysis: false, + media_type: deno_ast::MediaType::TypeScript, + maybe_syntax: None, + text: deno_core::ModuleCodeString::from(expr).into(), + })?; + Ok(parsed + .transpile( + &Default::default(), + &Default::default(), + &Default::default(), + )? + .into_source() + .text) +} + +pub fn get_annotation(inner_content: &str) -> NativeAnnotation { + let mut res = NativeAnnotation { useragent: None, proxy: None }; + + let anns = inner_content + .lines() + .take_while(|x| x.starts_with("//")) + .map(|x| x.to_string().trim_start_matches("//").trim().to_string()) + .collect_vec(); + + for ann in anns.iter() { + if ann.starts_with("useragent") { + res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); + } else if ann.starts_with("proxy") { + res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); + } + } + res +} + +fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { + RE_PROXY.captures(s).map(|x| { + ( + format!( + "{}://{}{}", + x.get(1).map(|x| x.as_str()).unwrap_or_default(), + x.get(5).map(|x| x.as_str()).unwrap_or_default(), + x.get(7) + .map(|x| format!(":{}", x.as_str())) + .unwrap_or_default(), + ), + x.get(3).map(|y| { + ( + y.as_str().to_string(), + x.get(4).map(|x| x.as_str().to_string()).unwrap_or_default(), + ) + }), + ) + }) +} + +fn write_error_expr(expr: &str, uuid: &Uuid) { + if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { + tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + return; + } + let dir_entries = match std::fs::read_dir(ERROR_DIR) { + Ok(entries) => entries.count(), + Err(_) => { + tracing::error!("failed to read error dir {ERROR_DIR}"); + return; + } + }; + + if std::env::var("PRINT_NATIVE_ERRORS").is_ok() { + tracing::info!("native error for job {uuid}: {expr}"); + } + if dir_entries >= 100 { + tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + return; + } + + let path = format!("/{uuid}.js"); + tracing::info!( + "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + ); + if let Err(e) = write_file(ERROR_DIR, &path, expr) { + tracing::error!("failed to write error expr to file {path}: {e}"); + } +} + +use windmill_common::utils::unsafe_raw; + +async fn append_result_stream( + conn: &Connection, + workspace_id: &str, + job_id: &Uuid, + nstream: &str, + offset: i32, +) -> windmill_common::error::Result<()> { + match conn { + Connection::Sql(db) => { + append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?; + } + Connection::Http(client) => { + #[derive(serde::Serialize)] + struct ResultStreamBody<'a> { + result_stream: &'a str, + offset: i32, + } + let body = ResultStreamBody { result_stream: nstream, offset }; + if let Err(e) = client + .post::<_, String>( + &format!( + "/api/w/{}/agent_workers/push_result_stream/{}", + workspace_id, job_id + ), + None, + &body, + ) + .await + { + tracing::error!(%job_id, %e, "error sending result stream for job {job_id}: {e}"); + } + } + } + Ok(()) +} + +// ── ops ────────────────────────────────────────────────────────────── + +#[op2] +#[serde] +fn op_get_static_args(op_state: Rc>) -> Vec> { + op_state + .borrow() + .borrow::() + .args + .iter() + .map(|x| x.as_ref().map(|y| y.get().to_string())) + .collect_vec() +} + +#[op2(fast)] +fn op_log(op_state: Rc>, #[string] log: &str) { + if let Err(e) = op_state + .borrow_mut() + .borrow_mut::() + .s + .send(log.to_string()) + { + tracing::error!("failed to send log: {e}"); + } +} + +// ── eval_fetch_timeout ─────────────────────────────────────────────── + +/// Execute a NativeTS script using deno_core/V8. +/// +/// Returns `(result, has_stream)` where `has_stream` indicates if the result +/// came from an async iterable stream. +/// +/// `otel_initialized` should be `DENO_OTEL_INITIALIZED.load(SeqCst)`. +/// +/// The caller (windmill-worker) is responsible for wrapping this in +/// `run_future_with_polling_update_job_poller` for job cancellation/polling. +#[allow(clippy::too_many_arguments)] +pub async fn eval_fetch_timeout( + env_code: String, + ts_expr: String, + js_expr: String, + args: Option<&Json>>>, + script_entrypoint_override: Option, + job_id: Uuid, + conn: &Connection, + w_id: &str, + load_client: bool, + otel_initialized: bool, + stream_notifier_update: Option>, +) -> windmill_common::error::Result<(Box, bool)> { + let (sender, mut receiver) = oneshot::channel::(); + let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); + let (result_stream_sender, mut result_stream_receiver) = mpsc::unbounded_channel::(); + + let conn_ = conn.clone(); + let w_id_ = w_id.to_string(); + tokio::spawn(async move { + while let Some(log) = append_logs_receiver.recv().await { + windmill_queue::append_logs(&job_id, &w_id_, log, &conn_).await + } + }); + + let append_result_stream_fn = append_result_stream; + let conn_ = conn.clone(); + let w_id_ = w_id.to_string(); + tokio::spawn(async move { + let mut offset = -1; + while let Some(stream) = result_stream_receiver.recv().await { + offset += 1; + if let Err(e) = append_result_stream_fn(&conn_, &w_id_, &job_id, &stream, offset).await + { + tracing::error!("failed to append result stream: {e}"); + } + } + }); + + let parsed_args = windmill_parser_ts::parse_deno_signature( + &ts_expr, + true, + false, + script_entrypoint_override.clone(), + )? + .args; + let spread = parsed_args + .into_iter() + .map(|x| { + args.as_ref() + .and_then(|args| args.0.get(&x.name).map(|x| x.clone())) + }) + .collect::>(); + + let ann = get_annotation(&ts_expr); + + #[cfg(not(feature = "enterprise"))] + if ann.proxy.is_some() { + return Err(Error::ExecutionErr("Proxy is an EE feature".to_string()).into()); + } + + let mut extra_logs = String::new(); + if ann.useragent.is_some() { + extra_logs.push_str(&format!("useragent: {}\n", ann.useragent.as_ref().unwrap())); + } + if ann.proxy.is_some() { + let (proxy, auth) = ann.proxy.as_ref().unwrap(); + extra_logs.push_str(&format!( + "proxy: {proxy} (basic auth: {})\n", + auth.is_some() + )); + } + + let result_f = tokio::task::spawn_blocking(move || { + let ops = vec![op_get_static_args(), op_log()]; + let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; + + let fetch_options = deno_fetch::Options { + root_cert_store_provider: None, + user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), + proxy: ann.proxy.map(|x| deno_tls::Proxy { + url: x.0, + basic_auth: x + .1 + .map(|(username, password)| deno_tls::BasicAuth { username, password }), + }), + ..Default::default() + }; + + let exts: Vec = vec![ + deno_telemetry::deno_telemetry::init_ops(), + deno_webidl::deno_webidl::init_ops(), + deno_url::deno_url::init_ops(), + deno_console::deno_console::init_ops(), + deno_web::deno_web::init_ops::( + Arc::new(BlobStore::default()), + None, + ), + deno_fetch::deno_fetch::init_ops::(fetch_options), + deno_net::deno_net::init_ops::(None, None), + ext, + ]; + + let options = RuntimeOptions { + is_main: true, + extensions: exts, + create_params: Some( + deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128), + ), + startup_snapshot: Some(RUNTIME_SNAPSHOT), + module_loader: Some(Rc::new(deno_core::FsModuleLoader)), + extension_transpiler: None, + ..Default::default() + }; + + let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>(); + + let mut js_runtime: JsRuntime = JsRuntime::new(options); + + // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized. + if otel_initialized { + if let Err(e) = + js_runtime.execute_script("", "globalThis.__bootstrapOtel()") + { + tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); + } + } + + js_runtime.add_near_heap_limit_callback(move |x, y| { + tracing::error!("heap limit reached: {x} {y}"); + if memory_limit_tx.send(()).is_err() { + tracing::error!("failed to send memory limit reached notification - isolate may already be terminating"); + }; + y * 2 + }); + + let (log_sender, mut log_receiver) = mpsc::unbounded_channel::(); + + { + let op_state = js_runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(PermissionsContainer {}); + op_state.put(MainArgs { args: spread }); + op_state.put(LogString { s: log_sender }); + } + + sender + .send(js_runtime.v8_isolate().thread_safe_handle()) + .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let future = async { + if !extra_logs.is_empty() { + if let Err(e) = append_logs_sender.send(extra_logs) { + tracing::error!("failed to send extra logs: {e}"); + } + } + let handle = tokio::spawn(async move { + let mut result_stream = String::new(); + let mut is_stream = false; + while let Some(log) = log_receiver.recv().await { + use windmill_common::result_stream::extract_stream_from_logs; + + if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { + if !is_stream { + is_stream = true; + if let Some(ref f) = stream_notifier_update { + f(); + } + } + + result_stream.push_str(&stream); + if let Err(e) = result_stream_sender.send(stream) { + tracing::error!("failed to send result stream: {e}"); + } + } else { + if let Err(e) = append_logs_sender.send(log) { + tracing::error!("failed to send log: {e}"); + } + } + } + if !result_stream.is_empty() { + Some(result_stream) + } else { + None + } + }); + + let r = tokio::select! { + r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id, otel_initialized) => Ok(r), + _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string())) + }; + drop(js_runtime); + if let Ok(r) = r { + match handle.await { + Ok(Some(logs)) => { + // merge_result_stream: if main result is null but stream exists, use stream + match r { + Ok(raw) if raw.get() == "null" => Ok((unsafe_raw(logs), true)), + Ok(raw) => Ok((raw, true)), + Err(e) => Err(e), + } + } + Ok(None) => Ok(r.map(|r| (r, false))?), + Err(e) => Err(Error::ExecutionErr(e.to_string())), + } + } else { + r.map(|r| r.map(|r| (r, false)))? + } + }; + let r = runtime.block_on(future)?; + + Ok(r) as windmill_common::error::Result<(Box, bool)> + }); + + let result = result_f.await.map_err(windmill_common::error::to_anyhow)?; + match result { + Ok((res, has_stream)) => Ok((res, has_stream)), + Err(e) => { + if let Ok(isolate) = receiver.try_recv() { + isolate.terminate_execution(); + } + Err(e) + } + } +} + +async fn eval_fetch( + js_runtime: &mut JsRuntime, + expr: &str, + env_code: Option, + script_entrypoint_override: Option, + load_client: bool, + job_id: &Uuid, + _otel_initialized: bool, +) -> windmill_common::error::Result> { + if load_client { + if let Some(env_code) = env_code.as_ref() { + let _ = js_runtime + .load_side_es_module_from_code( + &deno_core::resolve_url("file:///windmill.ts") + .map_err(windmill_common::error::to_anyhow)?, + format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()), + ) + .await + .map_err(windmill_common::error::to_anyhow)?; + } + } + use anyhow::Context; + use deno_core::error::CoreError; + use windmill_common::worker::to_raw_value; + let source = format!("{}\n{expr}", env_code.unwrap_or_default()); + let _ = js_runtime + .load_side_es_module_from_code( + &deno_core::resolve_url("file:///eval.ts") + .map_err(windmill_common::error::to_anyhow)?, + source.to_string(), + ) + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("failed to load module")?; + + let main_override = script_entrypoint_override.unwrap_or("main".to_string()); + + #[cfg(all(feature = "private", feature = "enterprise"))] + let otel_context_inject = if _otel_initialized { + let trace_id = job_id.as_simple().to_string(); + format!( + r#"globalThis.__enterSpan?.({{ + isRecording: () => true, + spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) +}});"# + ) + } else { + String::new() + }; + + #[cfg(not(all(feature = "private", feature = "enterprise")))] + let otel_context_inject = ""; + + let script = js_runtime + .execute_script( + "", + format!( + r#" +function isAsyncIterable(obj) {{ + return obj != null && typeof obj[Symbol.asyncIterator] === 'function'; +}} + +function processStreamIterative(res) {{ + const iterator = res[Symbol.asyncIterator](); + + function processLoop() {{ + return new Promise(function(resolve) {{ + function step() {{ + iterator.next().then(function(result) {{ + if (!result.done) {{ + const chunk = result.value; + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); + step(); + }} else {{ + resolve("null"); + }} + }}).catch(function(error) {{ + resolve("null"); + }}); + }} + step(); + }}); + }} + + return processLoop(); +}} + +{otel_context_inject} + +let args = Deno.core.ops.op_get_static_args().map(JSON.parse) +import("file:///eval.ts").then((module) => module.{main_override}(...args)) + .then(res => {{ + if (isAsyncIterable(res)) {{ + return processStreamIterative(res) + }} else {{ + return JSON.stringify(res ?? null); + }} + }}) +"# + ), + ) + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("native script initialization")?; + + let fut = js_runtime.resolve(script); + let global = js_runtime + .with_event_loop_promise(fut, PollEventLoopOptions::default()) + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }); + + match global { + Ok(global) => { + let scope = &mut js_runtime.handle_scope(); + let local = v8::Local::new(scope, global); + let r = serde_v8::from_v8::>(scope, local) + .map_err(windmill_common::error::to_anyhow)?; + Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) + } + Err(CoreError::Js(e)) => { + let stack_head = e.frames.first().and_then(|f| { + if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") { + Some(format!( + "{}\n", + source + .lines() + .nth((f.line_number.unwrap_or(1)) as usize - 1) + .unwrap_or("") + .to_string() + )) + } else { + None + } + }); + let stack_s = format!( + "{}{}", + stack_head.unwrap_or("".to_string()), + e.stack.unwrap_or("".to_string()) + ); + let stack = if stack_s.is_empty() { + None + } else { + Some(stack_s) + }; + Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({ + "message": e.message, + "stack": stack, + "name": e.name, + })))) + } + Err(e) => Err(Error::ExecutionErr(e.print_with_cause())), + } +} diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js new file mode 100644 index 0000000000..2639353a04 --- /dev/null +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -0,0 +1,62 @@ +import * as abortSignal from "ext:deno_web/03_abort_signal.js"; +import * as base64 from "ext:deno_web/05_base64.js"; +import * as console from "ext:deno_console/01_console.js"; +import * as encoding from "ext:deno_web/08_text_encoding.js"; +import * as event from "ext:deno_web/02_event.js"; +import * as fetch from "ext:deno_fetch/26_fetch.js"; +import * as file from "ext:deno_web/09_file.js"; +import * as fileReader from "ext:deno_web/10_filereader.js"; +import * as formData from "ext:deno_fetch/21_formdata.js"; +import * as headers from "ext:deno_fetch/20_headers.js"; +import * as streams from "ext:deno_web/06_streams.js"; +import * as timers from "ext:deno_web/02_timers.js"; +import * as url from "ext:deno_url/00_url.js"; +import * as net from "ext:deno_net/01_net.js"; +import * as tls from "ext:deno_net/02_tls.js"; +import * as urlPattern from "ext:deno_url/01_urlpattern.js"; +import * as webidl from "ext:deno_webidl/00_webidl.js"; +import * as response from "ext:deno_fetch/23_response.js"; +import * as request from "ext:deno_fetch/23_request.js"; +import "ext:deno_web/02_structured_clone.js"; +import "ext:deno_web/04_global_interfaces.js"; +import "ext:deno_web/13_message_port.js"; +import "ext:deno_web/14_compression.js"; +import "ext:deno_web/15_performance.js"; +import "ext:deno_web/16_image_data.js"; +import "ext:deno_fetch/27_eventsource.js"; + +globalThis.atob = base64.atob; +globalThis.btoa = base64.btoa; +globalThis.fetch = fetch.fetch; +globalThis.Request = request.Request; +globalThis.Response = response.Response; +globalThis.Blob = file.Blob; +globalThis.URL = url.URL; +globalThis.FormData = formData.FormData; +globalThis.URLSearchParams = url.URLSearchParams; +globalThis.Headers = headers.Headers; +globalThis.FileReader = fileReader.FileReader; +globalThis.console = new console.Console((msg, level) => + globalThis.Deno.core.ops.op_log(msg) +); +globalThis.AbortController = abortSignal.AbortController; +globalThis.AbortSignal = abortSignal.AbortSignal; + +Object.assign(globalThis, { + clearInterval: timers.clearInterval, + clearTimeout: timers.clearTimeout, + setInterval: timers.setInterval, + setTimeout: timers.setTimeout, +}); + +// Expose bootstrapOtel globally so it can be called from Rust after runtime creation. +// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. +// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] +// consoleConfig: 0=ignore, 1=capture, 2=replace +globalThis.__bootstrapOtel = () => { + import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { + bootstrap([1, 0, 1, 0]); + // Expose enterSpan for setting parent trace context + globalThis.__enterSpan = enterSpan; + }); +}; diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js similarity index 100% rename from backend/windmill-worker/src/windmill-client.js rename to backend/windmill-runtime-nativets/src/windmill-client.js diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 51df8ca7f9..4735838c03 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -23,9 +23,7 @@ parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] -deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", - "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile", - "quickjs"] +deno_core = ["dep:windmill-runtime-nativets"] libffi_mac = ["dep:libffi-sys"] otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"] dind = ["dep:bollard"] @@ -39,7 +37,6 @@ nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] ruby = ["dep:windmill-parser-ruby"] duckdb = ["dep:libloading"] -quickjs = ["dep:rquickjs"] bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"] [dependencies] @@ -47,6 +44,8 @@ windmill-queue.workspace = true windmill-dep-map.workspace = true windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. windmill-common = { workspace = true, default-features = false } +windmill-jseval.workspace = true +windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } windmill-macros.workspace = true windmill-parser.workspace = true @@ -97,20 +96,6 @@ once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true url.workspace = true -deno_telemetry = { workspace = true, optional = true } -deno_fetch = { workspace = true, optional = true } -deno_webidl = { workspace = true, optional = true } -deno_web = { workspace = true, optional = true } -deno_net = { workspace = true, optional = true } -deno_console = { workspace = true, optional = true } -deno_url = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } -deno_ast = { workspace = true, optional = true } -deno_tls = { workspace = true, optional = true } -deno_permissions = { workspace = true, optional = true } -deno_io = { workspace = true, optional = true } -deno_error = { workspace = true, optional = true } -rustls-pemfile = { workspace = true, optional = true } async-stream.workspace = true postgres-native-tls.workspace = true @@ -137,7 +122,6 @@ object_store = { workspace = true, optional = true} convert_case.workspace = true yaml-rust.workspace = true backon.workspace = true -winapi = { workspace = true, optional = true } pep440_rs.workspace = true process-wrap.workspace = true async-once-cell.workspace = true @@ -150,7 +134,6 @@ prost.workspace = true axum.workspace = true bollard = { workspace = true, optional = true } oracle = { workspace = true, optional = true } -rquickjs = { workspace = true, optional = true } hudsucker.workspace = true hyper-http-proxy.workspace = true hyper-tls.workspace = true @@ -158,18 +141,4 @@ hyper-util.workspace = true rcgen.workspace = true [build-dependencies] -deno_fetch = { workspace = true, optional = true } -deno_webidl = { workspace = true, optional = true } -deno_web = { workspace = true, optional = true } -deno_net = { workspace = true, optional = true } -deno_console = { workspace = true, optional = true } -deno_url = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } -deno_ast = { workspace = true, optional = true } -deno_tls = { workspace = true, optional = true } -deno_permissions = { workspace = true, optional = true } -deno_io = { workspace = true, optional = true } -deno_runtime = { workspace = true, optional = true } -deno_telemetry = { workspace = true, optional = true } -winapi = { workspace = true, optional = true } libffi-sys = { workspace = true, optional = true } diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 8534e10edf..f328e4d9d0 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -1,154 +1 @@ -#[cfg(feature = "deno_core")] -use deno_fetch::FetchPermissions; -#[cfg(feature = "deno_core")] -use deno_net::NetPermissions; -#[cfg(feature = "deno_core")] -use deno_web::{BlobStore, TimersPermission}; -#[cfg(feature = "deno_core")] -use std::borrow::Cow; -#[cfg(feature = "deno_core")] -use std::env; -#[cfg(feature = "deno_core")] -use std::io::Write; -#[cfg(feature = "deno_core")] -use std::path::{Path, PathBuf}; -#[cfg(feature = "deno_core")] -use std::sync::Arc; - -// #[cfg(feature = "deno_core")] -pub struct PermissionsContainer; - -#[cfg(feature = "deno_core")] -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _resolved: bool, - _p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } -} - -#[cfg(feature = "deno_core")] -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -#[cfg(feature = "deno_core")] -impl NetPermissions for PermissionsContainer { - fn check_read<'a>( - &mut self, - _p: &'a str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_write<'a>( - &mut self, - _p: &'a str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_write_path<'a>( - &mut self, - _: &'a Path, - _: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - todo!() - } -} - -#[cfg(feature = "deno_core")] -deno_core::extension!( - fetch, - esm_entry_point = "ext:fetch/src/runtime.js", - esm = ["src/runtime.js"], -); - -#[cfg(feature = "deno_core")] -fn main() { - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); - println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - - let exts = vec![ - deno_telemetry::deno_telemetry::init_ops_and_esm(), - deno_webidl::deno_webidl::init_ops_and_esm(), - deno_url::deno_url::init_ops_and_esm(), - deno_console::deno_console::init_ops_and_esm(), - deno_web::deno_web::init_ops_and_esm::( - Arc::new(BlobStore::default()), - None, - ), - deno_fetch::deno_fetch::init_ops_and_esm::(Default::default()), - deno_net::deno_net::init_ops_and_esm::(None, None), - fetch::init_ops_and_esm(), - ]; - - // Build the file path to the snapshot. - let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); - - // Create the snapshot. - let output = deno_core::snapshot::create_snapshot( - deno_core::snapshot::CreateSnapshotOptions { - cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), - startup_snapshot: None, - extension_transpiler: Some(std::rc::Rc::new(|specifier, source| { - deno_runtime::transpile::maybe_transpile_source(specifier, source) - })), - extensions: exts, - with_runtime_cb: None, - skip_op_registration: false, - }, - None, - ) - .unwrap(); - - // NOTE(bartlomieju): Compressing the TSC snapshot in debug build took - // ~45s on M1 MacBook Pro; without compression it took ~1s. - // Thus we're not using compressed snapshot, trading off - // a lot of build time for some startup time in debug build. - let mut file = std::fs::File::create(snapshot_path).unwrap(); - // if cfg!(debug_assertions) { - file.write_all(&output.output).unwrap(); - // } else { - // let mut vec = Vec::with_capacity(output.output.len()); - // vec.extend((output.output.len() as u32).to_le_bytes()); - // vec.extend_from_slice( - // &zstd::bulk::compress(&output.output, 22).expect("snapshot compression failed"), - // ); - // file.write_all(&vec).unwrap(); - // } - - for path in output.files_loaded_during_snapshot { - println!("cargo:rerun-if-changed={}", path.display()); - } -} - -#[cfg(not(feature = "deno_core"))] fn main() {} diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 5efd29ff3c..38183ed774 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -333,10 +333,7 @@ pub async fn read_file_bytes(path: &str) -> error::Result> { Ok(content) } -//this skips more steps than from_str at the cost of being unsafe. The source must ALWAUS gemerate valid json or this can cause UB in the worst case -pub fn unsafe_raw(json: String) -> Box { - unsafe { std::mem::transmute::, Box>(json.into()) } -} +pub use windmill_common::utils::unsafe_raw; fn check_result_too_big(size: usize) -> error::Result<()> { if *CLOUD_HOSTED && size > MAX_RESULT_SIZE { diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 0df576d9b8..5ee3b34cf9 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -6,355 +6,23 @@ * LICENSE-AGPL for a copy of the license. */ -#[cfg(feature = "deno_core")] -use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc}; - use std::{collections::HashMap, sync::Arc}; -#[cfg(feature = "deno_core")] -use deno_ast::ParseParams; -#[cfg(feature = "deno_core")] -use deno_core::{ - op2, serde_v8, url, - v8::{self, IsolateHandle}, - Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, -}; -#[cfg(feature = "deno_core")] -use deno_fetch::FetchPermissions; -#[cfg(feature = "deno_core")] -use deno_net::NetPermissions; - -#[cfg(feature = "deno_core")] -use deno_web::{BlobStore, TimersPermission}; -#[cfg(feature = "deno_core")] -use itertools::Itertools; -use lazy_static::lazy_static; -#[cfg(feature = "quickjs")] -use once_cell::sync::Lazy; -use regex::Regex; use serde_json::value::RawValue; use sqlx::types::Json; - -#[cfg(feature = "deno_core")] -use tokio::{ - sync::{mpsc, oneshot}, - time::timeout, -}; use uuid::Uuid; -#[cfg(feature = "deno_core")] -use windmill_common::error::Error; -#[cfg(feature = "deno_core")] -use windmill_common::utils::configure_client; -#[cfg(feature = "deno_core")] -use windmill_common::worker::{write_file, TMP_DIR}; - -use windmill_common::flow_status::JobResult; +use windmill_common::client::AuthedClient; +use windmill_common::worker::Connection; use windmill_queue::CanceledBy; use crate::common::{OccupancyMetrics, StreamNotifier}; -use windmill_common::client::AuthedClient; #[cfg(feature = "deno_core")] -use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller}; +use crate::handle_child::run_future_with_polling_update_job_poller; -#[derive(Debug, Clone)] -pub struct IdContext { - pub flow_job: Uuid, - #[allow(dead_code)] - pub steps_results: HashMap, - pub previous_id: String, -} - -// #[cfg(feature = "deno_core")] -// pub struct ContainerRootCertStoreProvider { -// root_cert_store: RootCertStore, -// } - -// #[cfg(feature = "deno_core")] -// impl ContainerRootCertStoreProvider { -// fn new() -> ContainerRootCertStoreProvider { -// return ContainerRootCertStoreProvider { -// root_cert_store: deno_tls::create_default_root_cert_store(), -// }; -// } - -// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { -// let cert_file = std::fs::File::open(cert_path)?; -// let mut reader = BufReader::new(cert_file); -// let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; - -// self.root_cert_store.add_parsable_certificates(pem_file); -// Ok(()) -// } -// } - -// #[cfg(feature = "deno_core")] -// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { -// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { -// Ok(&self.root_cert_store) -// } -// } - -#[cfg(feature = "deno_core")] -pub struct PermissionsContainer; - -#[cfg(feature = "deno_core")] -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - Ok(()) - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _resolved: bool, - p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_io::fs::FsError> { - Ok(Cow::Borrowed(p)) - } -} - -#[cfg(feature = "deno_core")] -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -#[cfg(feature = "deno_core")] -impl NetPermissions for PermissionsContainer { - fn check_read<'a>( - &mut self, - p: &'a str, - _api_name: &str, - ) -> Result { - Ok(PathBuf::from(p)) - } - - fn check_write<'a>( - &mut self, - p: &'a str, - _api_name: &str, - ) -> Result { - Ok(PathBuf::from(p)) - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - Ok(()) - } - - fn check_write_path<'a>( - &mut self, - p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - Ok(Cow::Borrowed(p)) - } -} - -#[cfg(feature = "deno_core")] -pub struct OptAuthedClient(Option); - -const FLOW_INPUT_PREFIX: &'static str = "flow_input"; -const ENV_KEY_PREFIX: &'static str = "flow_env"; -const DOT_PATTERN: &'static str = "."; -const START_BRACKET_PATTERN: &'static str = "[\""; -const END_BRACKET_PATTERN: &'static str = "\"]"; - -/// Determines if we should prepend "return" to the expression -#[cfg(feature = "deno_core")] -fn should_add_return(expr: &str) -> bool { - // Trim whitespace - let trimmed = expr.trim(); - - // If it's empty, add return - if trimmed.is_empty() { - return true; - } - - // Check if it already starts with 'return' keyword (as a statement) - // Use word boundary to avoid matching "return" in variable names - if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { - return false; - } - - // Check for common statement patterns that shouldn't have return prepended - let statement_prefixes = [ - "const ", - "let ", - "var ", - "if ", - "if(", - "for ", - "for(", - "while ", - "while(", - "switch ", - "switch(", - "try ", - "try{", - "throw ", - "function ", - "class ", - "async ", - "await ", - ]; - - for prefix in &statement_prefixes { - if trimmed.starts_with(prefix) { - return false; - } - } - - // Check for multiple statements (contains semicolon not in a string) - // This is still not perfect but better than current logic - if contains_semicolon_outside_strings(trimmed) { - return false; - } - - // Default: assume it's an expression that needs return - true -} - -/// Checks if the expression contains a semicolon outside of strings -#[cfg(feature = "deno_core")] -fn contains_semicolon_outside_strings(expr: &str) -> bool { - let mut in_single_quote = false; - let mut in_double_quote = false; - let mut in_template = false; - let mut prev_char = '\0'; - - for ch in expr.chars() { - match ch { - '\'' if prev_char != '\\' && !in_double_quote && !in_template => { - in_single_quote = !in_single_quote; - } - '"' if prev_char != '\\' && !in_single_quote && !in_template => { - in_double_quote = !in_double_quote; - } - '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { - in_template = !in_template; - } - ';' if !in_single_quote && !in_double_quote && !in_template => { - return true; - } - _ => {} - } - prev_char = ch; - } - - false -} - -fn try_exact_property_access( - expr: &str, - flow_input: Option<&mappable_rc::Marc>>>, - flow_env: Option<&HashMap>>, -) -> Option> { - let obj = if expr.starts_with(FLOW_INPUT_PREFIX) { - Some(( - FLOW_INPUT_PREFIX, - flow_input.as_ref().map(|obj| obj.as_ref()), - )) - } else if expr.starts_with(ENV_KEY_PREFIX) { - Some((ENV_KEY_PREFIX, flow_env)) - } else { - None - }; - - if let Some((prefix, obj)) = obj { - let access_pattern_pos = prefix.len(); - let suffix = &expr[access_pattern_pos..]; - let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { - let key_name_pos = DOT_PATTERN.len(); - Some(&suffix[key_name_pos..]) - } else if suffix.starts_with(START_BRACKET_PATTERN) { - let key_name_pos = START_BRACKET_PATTERN.len(); - let suffix = &suffix[key_name_pos..]; - - let flow_arg_name = suffix - .ends_with(END_BRACKET_PATTERN) - .then(|| { - let start_key_name_pos = access_pattern_pos + key_name_pos; - let end_key_name_pos = expr.len() - END_BRACKET_PATTERN.len(); - &expr[start_key_name_pos..end_key_name_pos] - }) - .filter(|s| s.len() > 0); - flow_arg_name - } else { - None - }; - - if let Some(key_name) = maybe_key_name { - if let Some(key_value) = obj.and_then(|obj| obj.get(key_name)) { - return Some(key_value.clone()); - } - } - } - None -} - -async fn handle_full_regex( - expr: &str, - authed_client: &AuthedClient, - by_id: &IdContext, -) -> Option>> { - if let Some(captures) = RE_FULL.captures(&expr) { - let obj_name = captures.get(1).unwrap().as_str(); - let obj_key = captures.get(2).unwrap().as_str(); - let idx_o = captures.get(3).map(|y| y.as_str()); - let rest = captures.get(4).map(|y| y.as_str()); - let query = if let Some(idx) = idx_o { - match rest { - Some(rest) => Some(format!("{}{}", idx, rest)), - None => Some(idx.to_string()), - } - } else { - rest.map(|x| x.trim_start_matches('.').to_string()) - }; - - let result = if obj_name == "results" { - // Use .ok() to match deno_core op_get_id behavior: return null for non-existent steps - // instead of throwing an error - let res = authed_client - .get_result_by_id::>>( - &by_id.flow_job.to_string(), - obj_key, - query, - ) - .await - .ok() - .flatten(); - match res { - Some(v) => Ok(v), - None => serde_json::value::to_raw_value(&serde_json::Value::Null) - .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), - } - } else if obj_name == "flow_env" { - authed_client - .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) - .await - } else { - unreachable!(); - }; - - return Some(result); - } - - return None; -} +// Re-export IdContext from windmill-jseval for backward compatibility +pub use windmill_jseval::IdContext; pub async fn eval_timeout( expr: String, @@ -363,7 +31,7 @@ pub async fn eval_timeout( flow_env: Option<&HashMap>>, authed_client: Option<&AuthedClient>, by_id: Option<&IdContext>, - #[allow(unused_variables)] ctx: Option>, + ctx: Option>, ) -> anyhow::Result> { let expr = expr.trim().to_string(); @@ -377,7 +45,9 @@ pub async fn eval_timeout( return Ok(value.as_ref().to_owned()); } - if let Some(value) = try_exact_property_access(&expr, flow_input.as_ref(), flow_env) { + if let Some(value) = + windmill_jseval::try_exact_property_access(&expr, flow_input.as_ref(), flow_env) + { return Ok(value); } @@ -394,7 +64,6 @@ pub async fn eval_timeout( && transform_context.contains_key("previous_result") && p_ids.as_ref().unwrap().iter().any(|x| x == &expr) { - // tracing::error!("PREVIOUS_RESULT"); return Ok(transform_context .get("previous_result") .unwrap() @@ -403,571 +72,27 @@ pub async fn eval_timeout( } if let (Some(by_id), Some(authed_client)) = (by_id, authed_client) { - if let Some(result) = handle_full_regex(&expr, authed_client, by_id).await { + if let Some(result) = windmill_jseval::handle_full_regex(&expr, authed_client, by_id).await + { return result; } } - // Use QuickJS if enabled and either deno_core is not available or USE_QUICKJS env var is set - #[cfg(all(feature = "quickjs", not(feature = "deno_core")))] - { - return crate::js_eval_quickjs::eval_timeout_quickjs( - expr, - transform_context, - flow_input, - flow_env, - authed_client, - by_id, - ctx, - ) - .await; - } - - #[cfg(all(feature = "quickjs", feature = "deno_core"))] - if *USE_QUICKJS { - return crate::js_eval_quickjs::eval_timeout_quickjs( - expr, - transform_context, - flow_input, - flow_env, - authed_client, - by_id, - ctx, - ) - .await; - } - - #[cfg(not(feature = "deno_core"))] - { - #[allow(unreachable_code)] - return Err(anyhow::anyhow!("Deno core is not enabled".to_string()).into()); - } - - #[cfg(feature = "deno_core")] - { - let expr2 = expr.clone(); - let by_id = by_id.cloned(); - let (sender, mut receiver) = oneshot::channel::(); - let has_client = authed_client.is_some(); - let authed_client = authed_client.cloned(); - return timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - let mut ops = vec![op_get_context()]; - - if authed_client.is_some() { - ops.extend([ - // An op for summing an array of numbers - // The op-layer automatically deserializes inputs - // and serializes the returned Result & value - op_variable(), - op_resource(), - ]) - } - - if by_id.is_some() && authed_client.is_some() { - ops.push(op_get_result()); - ops.push(op_get_id()); - ops.push(op_get_flow_env()); - } - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - extensions: exts, - // startup_snapshot: Some(Snapshot::Static(buffer)), - ..Default::default() - }; - - let mut context_keys = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .map(|x| x.clone()) - .collect_vec(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - // tracing::error!("PREVIOUS_RESULT"); - context_keys.push("previous_result".to_string()); - } - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - let mut js_runtime = JsRuntime::new(options); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - let mut client = authed_client.clone(); - if let Some(client) = client.as_mut() { - client.force_client = Some( - configure_client( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .danger_accept_invalid_certs( - std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - ), - ) - .build() - .unwrap(), - ); - } - op_state.put(OptAuthedClient(client)); - op_state.put(TransformContext { - flow_input: if has_flow_input { flow_input } else { None }, - envs: transform_context - .into_iter() - .filter(|(a, _)| context_keys.contains(a)) - .collect(), - }); - } - - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| { - Error::ExecutionErr("impossible to send v8 isolate".to_string()) - })?; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - // pretty frail but this it to make the expr more user friendly and not require the user to write await - let expr = ["variable", "resource"] - .into_iter() - .fold(expr, replace_with_await); - - let expr = replace_with_await_result(expr); - - let r = runtime.block_on(eval( - &mut js_runtime, - &expr, - context_keys, - by_id, - has_client, - ctx, - ))?; - - Ok(r) as anyhow::Result> - }), - ) - .await - .map_err(|_| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - }; - Error::ExecutionErr(format!( - "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" - )) - })??; - } -} - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -pub fn replace_with_await(expr: String, fn_name: &str) -> String { - let sep = format!("{}(", fn_name); - let mut split = expr.split(&sep); - let mut s = split.next().unwrap_or_else(|| "").to_string(); - for x in split { - s.push_str(&format!("(await {}({}", fn_name, add_closing_bracket(x))) - } - s -} -lazy_static! { - static ref RE: Regex = Regex::new( - r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + windmill_jseval::eval_timeout_quickjs( + expr, + transform_context, + flow_input, + flow_env, + authed_client, + by_id, + ctx, ) - .unwrap(); - static ref RE_FULL: Regex = Regex::new( - r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" - ) - .unwrap(); - static ref RE_PROXY: Regex = - Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); -} - -#[cfg(feature = "quickjs")] -#[allow(dead_code)] // Only used when both quickjs and deno_core features are enabled -static USE_QUICKJS: Lazy = Lazy::new(|| { - // When enterprise is not enabled, default to QuickJS unless USE_DENO_FOR_FLOW_EVAL is set - // When enterprise is enabled, default to Deno unless USE_QUICKJS_FOR_FLOW_EVAL is set - #[cfg(not(feature = "enterprise"))] - { - std::env::var("USE_DENO_FOR_FLOW_EVAL").is_err() - } - #[cfg(feature = "enterprise")] - { - std::env::var("USE_QUICKJS_FOR_FLOW_EVAL").is_ok() - } -}); - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -pub fn replace_with_await_result(expr: String) -> String { - RE.replace_all(&expr, "(await $r)").to_string() -} - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -fn add_closing_bracket(s: &str) -> String { - let mut s = s.to_string(); - let mut level = 1; - let mut idx = 0; - for c in s.chars() { - match c { - '(' => level += 1, - ')' => level -= 1, - _ => (), - }; - if level == 0 { - break; - } - idx += 1; - } - s.insert_str(idx, ")"); - s -} - -#[cfg(feature = "deno_core")] -async fn eval( - context: &mut JsRuntime, - expr: &str, - transform_context: Vec, - by_id: Option, - has_client: bool, - ctx: Option>, -) -> anyhow::Result> { - tracing::debug!("evaluating: {} {:#?}", expr, by_id); - - let (api_code, by_id_code) = if has_client { - let by_id_code = if let Some(by_id) = by_id { - format!( - r#" -async function result_by_id(node_id) {{ - let id_map = {{ {} }}; - let id = id_map[node_id]; - if (node_id == "{}") {{ - return previous_result; - }} else if (id) {{ - if (Array.isArray(id)) {{ - return await Promise.all(id.map(async (id) => await get_result(id))); - }} else {{ - return await get_result(id); - }} - }} else {{ - let flow_job_id = "{}"; - return JSON.parse(await Deno.core.ops.op_get_id(flow_job_id, node_id)); - }} -}} - -async function get_result(id) {{ - return JSON.parse(await Deno.core.ops.op_get_result(id)); -}} -const results = new Proxy({{}}, {{ - get: function(target, name, receiver) {{ - return result_by_id(name); - }} -}}); - -async function flow_env_by_var_name(var_name) {{ - let root_job_id = "{}"; - return JSON.parse(await Deno.core.ops.op_get_flow_env(root_job_id, var_name, null)); -}} - -const flow_env = new Proxy({{}}, {{ - get: function(target, name, receiver) {{ - return flow_env_by_var_name(name); - }} -}}); - -"#, - by_id - .steps_results - .into_iter() - .map(|(k, v)| { - let v_str = match v { - JobResult::SingleJob(x) => format!("\"{x}\""), - JobResult::ListJob(x) => { - format!("[{}]", x.iter().map(|x| format!("\"{x}\"")).join(",")) - } - }; - format!("\"{k}\": {v_str}") - }) - .join(","), - by_id.previous_id, - by_id.flow_job, - by_id.flow_job - ) - } else { - String::new() - }; - - let api_code = format!( - r#" -async function variable(path) {{ - return await Deno.core.ops.op_variable(path); -}} -async function resource(path) {{ - return JSON.parse(await Deno.core.ops.op_resource(path)); -}} - "#, - ); - (api_code, by_id_code) - } else { - (String::new(), String::new()) - }; - - let f = if should_add_return(expr) { - format!("return {expr}") - } else { - expr.to_string() - }; - - let ctx_str = ctx - .map(|x| { - x.into_iter() - .map(|(k, v)| format!("let {} = \"{}\";", k, v)) - .join("\n") - }) - .unwrap_or_default(); - let code = format!( - r#" -function get_from_env(name) {{ - return JSON.parse(Deno.core.ops.op_get_context(name)); -}} -{ctx_str} - -{api_code} -{} -{} -{by_id_code} -((async () => {{ - {f}; -}})()).then((x) => JSON.stringify(x ?? null)) - "#, - transform_context - .iter() - .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) }) - .join(""), - if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) { - r#"let error = previous_result?.error; -if (!error) { - if (Array.isArray(previous_result)) { - const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); - if (errors.length === 1) { - error = errors[0].error; - } else if (errors.length > 1) { - error = { - name: 'MultipleErrors', - message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), - errors: previous_result - }; - } else { - error = { - name: 'MultipleErrors', - message: "Could not parse errors", - errors: previous_result - }; - } - } else { - if (previous_result) { - error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; - } else { - error = { name: 'UnknownError', message: 'No error found' }; - } - } -}"# - } else { - "" - }, - ); - - let script = context.execute_script("", code)?; - let fut = context.resolve(script); - let global = context - .with_event_loop_promise(fut, PollEventLoopOptions::default()) - .await?; - - let scope = &mut context.handle_scope(); - let local = v8::Local::new(scope, global); - // Deserialize a `v8` object into a Rust type using `serde_v8`, - // in this case deserialize to a JSON `Value`. - let r = serde_v8::from_v8::(scope, local)?; - Ok(unsafe_raw(r)) -} - -// #[warn(dead_code)] -// async fn op_test( -// _state: Rc>, -// path: String, -// _buf: Option, -// ) -> Result { -// tokio::time::sleep(std::time::Duration::from_secs(1)).await; -// Ok(path) -// } - -// TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client? -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_variable( - op_state: Rc>, - #[string] path: String, -) -> Result { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - Ok(client - .get_variable_value(&path) - .await - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_result( - op_state: Rc>, - #[string] id: String, -) -> Result { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_completed_job_result::>(&id, None) - .await - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - .map(|x| x.get().to_string()) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_id( - op_state: Rc>, - #[string] flow_job_id: String, - #[string] node_id: String, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - let result = client - .get_result_by_id::>>(&flow_job_id, &node_id, None) - .await - .ok(); - if let Some(result) = result { - Ok(result.map(|x| x.get().to_string())) - } else { - Ok(None) - } - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_resource( - op_state: Rc>, - #[string] path: String, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_resource_value_interpolated::>>(&path, None) - .await - .map(|x| x.map(|x| x.get().to_string())) - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_flow_env( - op_state: Rc>, - #[string] root_job_id: String, - #[string] var_name: String, - #[string] json_path: Option, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_flow_env_by_flow_job_id::>>( - &root_job_id, - &var_name, - json_path, - ) - .await - .map(|value| value.map(|val| val.get().to_string())) - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -pub struct TransformContext { - pub envs: HashMap>>, - pub flow_input: Option>>>, -} - -#[cfg(feature = "deno_core")] -#[op2] -#[string] -fn op_get_context(op_state: Rc>, #[string] id: &str) -> String { - let ops = op_state.borrow(); - let client = ops.borrow::(); - if id == "flow_input" { - client - .flow_input - .as_ref() - .and_then(|x| serde_json::to_string(x.as_ref()).ok()) - .unwrap_or_else(|| "null".to_string()) - } else { - client - .envs - .get(id) - .and_then(|x| serde_json::to_string(x).ok()) - .unwrap_or_else(String::new) - } + .await } #[cfg(feature = "deno_core")] pub fn transpile_ts(expr: String) -> anyhow::Result { - let parsed = deno_ast::parse_module(ParseParams { - specifier: url::Url::parse("file:///eval.ts")?, - capture_tokens: false, - scope_analysis: false, - media_type: deno_ast::MediaType::TypeScript, - maybe_syntax: None, - text: deno_core::ModuleCodeString::from(expr).into(), - })?; - Ok(parsed - .transpile( - &Default::default(), - &Default::default(), - &Default::default(), - )? - .into_source() - .text) + windmill_runtime_nativets::transpile_ts(expr) } #[cfg(not(feature = "deno_core"))] @@ -975,68 +100,6 @@ pub fn transpile_ts(_expr: String) -> anyhow::Result { Ok("require deno".to_string()) } -#[cfg(feature = "deno_core")] -static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); - -#[cfg(feature = "deno_core")] -pub struct MainArgs { - args: Vec>>, -} - -#[cfg(feature = "deno_core")] -pub struct LogString { - pub s: mpsc::UnboundedSender, -} - -#[cfg(feature = "deno_core")] -pub struct NativeAnnotation { - pub useragent: Option, - pub proxy: Option<(String, Option<(String, String)>)>, -} -#[cfg(feature = "deno_core")] -pub fn get_annotation(inner_content: &str) -> NativeAnnotation { - let mut res = NativeAnnotation { useragent: None, proxy: None }; - - let anns = inner_content - .lines() - .take_while(|x| x.starts_with("//")) - .map(|x| x.to_string().trim_start_matches("//").trim().to_string()) - .collect_vec(); - - for ann in anns.iter() { - if ann.starts_with("useragent") { - res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); - } else if ann.starts_with("proxy") { - res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); - } - } - res -} - -#[cfg(feature = "deno_core")] -fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { - RE_PROXY.captures(s).map(|x| { - ( - format!( - "{}://{}{}", - x.get(1).map(|x| x.as_str()).unwrap_or_default(), - x.get(5).map(|x| x.as_str()).unwrap_or_default(), - x.get(7) - .map(|x| format!(":{}", x.as_str())) - .unwrap_or_default(), - ), - x.get(3).map(|y| { - ( - y.as_str().to_string(), - x.get(4).map(|x| x.as_str().to_string()).unwrap_or_default(), - ) - }), - ) - }) -} - -use windmill_common::worker::Connection; - #[cfg(not(feature = "deno_core"))] pub async fn eval_fetch_timeout( _env_code: String, @@ -1079,220 +142,37 @@ pub async fn eval_fetch_timeout( stream_notifier: Option, has_stream: &mut bool, ) -> windmill_common::error::Result> { - let (sender, mut receiver) = oneshot::channel::(); - let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); - let (result_stream_sender, mut result_stream_receiver) = mpsc::unbounded_channel::(); - - let conn_ = conn.clone(); - let w_id_ = w_id.to_string(); - tokio::spawn(async move { - while let Some(log) = append_logs_receiver.recv().await { - windmill_queue::append_logs(&job_id, &w_id_, log, &conn_).await - } - }); - - let conn_ = conn.clone(); - let w_id_ = w_id.to_string(); - tokio::spawn(async move { - let mut offset = -1; - while let Some(stream) = result_stream_receiver.recv().await { - use crate::job_logger::append_result_stream; - offset += 1; - if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream, offset).await { - tracing::error!("failed to append result stream: {e}"); - } - } - }); - let parsed_args = windmill_parser_ts::parse_deno_signature( - &ts_expr, - true, - false, - script_entrypoint_override.clone(), - )? - .args; - let spread = parsed_args - .into_iter() - .map(|x| { - args.as_ref() - .and_then(|args| args.0.get(&x.name).map(|x| x.clone())) - }) - .collect::>(); - - let ann = get_annotation(&ts_expr); - - #[cfg(not(feature = "enterprise"))] - if ann.proxy.is_some() { - return Err(Error::ExecutionErr("Proxy is an EE feature".to_string()).into()); - } - - let mut extra_logs = String::new(); - if ann.useragent.is_some() { - extra_logs.push_str(&format!("useragent: {}\n", ann.useragent.as_ref().unwrap())); - } - if ann.proxy.is_some() { - let (proxy, auth) = ann.proxy.as_ref().unwrap(); - extra_logs.push_str(&format!( - "proxy: {proxy} (basic auth: {})\n", - auth.is_some() - )); - } - - let result_f = tokio::task::spawn_blocking(move || { - let ops = vec![op_get_static_args(), op_log()]; - let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; - - let fetch_options = deno_fetch::Options { - root_cert_store_provider: None, - user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), - proxy: ann.proxy.map(|x| deno_tls::Proxy { - url: x.0, - basic_auth: x - .1 - .map(|(username, password)| deno_tls::BasicAuth { username, password }), - }), - ..Default::default() - }; - - let exts: Vec = vec![ - deno_telemetry::deno_telemetry::init_ops(), - deno_webidl::deno_webidl::init_ops(), - deno_url::deno_url::init_ops(), - deno_console::deno_console::init_ops(), - deno_web::deno_web::init_ops::( - Arc::new(BlobStore::default()), - None, - ), - deno_fetch::deno_fetch::init_ops::(fetch_options), - deno_net::deno_net::init_ops::(None, None), - ext, - ]; - - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - is_main: true, - extensions: exts, - create_params: Some( - deno_core::v8::CreateParams::default() - .heap_limits(0 as usize, 1024 * 1024 * 128 as usize), - ), - // startup_snapshot: None, - startup_snapshot: Some(RUNTIME_SNAPSHOT), - module_loader: Some(Rc::new(deno_core::FsModuleLoader)), - extension_transpiler: None, - ..Default::default() - }; - - let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>(); - - // tracing::info!("starting isolate"); - // let instant = Instant::now(); - - let mut js_runtime: JsRuntime = JsRuntime::new(options); - // tracing::info!("ttc: {:?}", instant.elapsed()); - - // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized. - // We call the function exposed by runtime.js since we can't dynamically import ext: modules. + let otel_initialized = { #[cfg(all(feature = "private", feature = "enterprise"))] - if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - if let Err(e) = - js_runtime.execute_script("", "globalThis.__bootstrapOtel()") - { - tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); - } + { + crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + false + } + }; - js_runtime.add_near_heap_limit_callback(move |x,y| { - tracing::error!("heap limit reached: {x} {y}"); - - if memory_limit_tx.send(()).is_err() { - tracing::error!("failed to send memory limit reached notification - isolate may already be terminating"); - }; - //to give a bit of time to kill the worker without v8 crashing - return y*2; + let stream_notifier_update: Option> = stream_notifier + .map(|sn| { + Arc::new(move || { + sn.update_flow_status_with_stream_job(); + }) as Arc }); - let (log_sender, mut log_receiver) = mpsc::unbounded_channel::(); - - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(PermissionsContainer {}); - //reqwest client seems to not be sharable between runtimes unfortunately - // op_state.put(HTTP_CLIENT.clone()); - op_state.put(MainArgs { args: spread }); - op_state.put(LogString { s: log_sender }); - } - - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let future = async { - use crate::common::merge_result_stream; - - if !extra_logs.is_empty() { - if let Err(e) = append_logs_sender.send(extra_logs) { - tracing::error!("failed to send extra logs: {e}"); - } - } - let handle = tokio::spawn(async move { - let mut result_stream = String::new(); - let mut is_stream = false; - while let Some(log) = log_receiver.recv().await { - use windmill_common::result_stream::extract_stream_from_logs; - - if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { - if let Some(sn) = stream_notifier.as_ref() { - if !is_stream { - is_stream = true; - sn.update_flow_status_with_stream_job(); - } - } - - result_stream.push_str(&stream); - if let Err(e) = result_stream_sender.send(stream) { - tracing::error!("failed to send result stream: {e}"); - } - } else { - if let Err(e) = append_logs_sender.send(log) { - tracing::error!("failed to send log: {e}"); - } - } - } - if !result_stream.is_empty() { - Some(result_stream) - } else { - None - } - }); - - let r = tokio::select! { - r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id) => Ok(r), - _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string())) - }; - drop(js_runtime); - if let Ok(r) = r { - match handle.await { - Ok(Some(logs)) => { - Ok(merge_result_stream(r, Some(logs)).await.map(|r| (r, true))) - } - Ok(None) => Ok(r.map(|r| (r, false))), - Err(e) => Err(Error::ExecutionErr(e.to_string())), - } - } else { - r.map(|r| r.map(|r| (r, false))) - } - // r - }; - let r = runtime.block_on(future)?; - // tracing::info!("total: {:?}", instant.elapsed()); - - r as windmill_common::error::Result<(Box, bool)> - }); + let result_f = windmill_runtime_nativets::eval_fetch_timeout( + env_code, + ts_expr, + js_expr, + args, + script_entrypoint_override, + job_id, + conn, + w_id, + load_client, + otel_initialized, + stream_notifier_update, + ); let (res, new_has_stream) = run_future_with_polling_update_job_poller( job_id, @@ -1300,476 +180,14 @@ pub async fn eval_fetch_timeout( conn, mem_peak, canceled_by, - async { result_f.await.map_err(windmill_common::error::to_anyhow)? }, + result_f, worker_name, w_id, &mut Some(occupation_metrics), Box::pin(futures::stream::once(async { 0 })), ) - .await - .map_err(|e| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - } - e - })?; + .await?; *has_stream = new_has_stream; *mem_peak = (res.get().len() / 1000) as i32; Ok(res) } - -#[cfg(feature = "deno_core")] -const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); - -#[cfg(feature = "deno_core")] -const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); - -#[cfg(feature = "deno_core")] -fn write_error_expr(expr: &str, uuid: &Uuid) { - if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { - tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); - return; - } - let dir_entries = match std::fs::read_dir(ERROR_DIR) { - Ok(entries) => entries.count(), - Err(_) => { - tracing::error!("failed to read error dir {ERROR_DIR}"); - return; - } - }; - - if std::env::var("PRINT_NATIVE_ERRORS").is_ok() { - tracing::info!("native error for job {uuid}: {expr}"); - } - if dir_entries >= 100 { - tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); - return; - } - - let path = format!("/{uuid}.js"); - tracing::info!( - "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" - ); - if let Err(e) = write_file(ERROR_DIR, &path, expr) { - tracing::error!("failed to write error expr to file {path}: {e}"); - } -} - -#[cfg(feature = "deno_core")] -async fn eval_fetch( - js_runtime: &mut JsRuntime, - expr: &str, - env_code: Option, - script_entrypoint_override: Option, - load_client: bool, - job_id: &Uuid, -) -> windmill_common::error::Result> { - if load_client { - if let Some(env_code) = env_code.as_ref() { - let _ = js_runtime - .load_side_es_module_from_code( - &deno_core::resolve_url("file:///windmill.ts").map_err(error::to_anyhow)?, - format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()), - ) - .await - .map_err(error::to_anyhow)?; - } - } - use anyhow::Context; - use deno_core::error::CoreError; - use windmill_common::{error, worker::to_raw_value}; - let source = format!("{}\n{expr}", env_code.unwrap_or_default()); - let _ = js_runtime - .load_side_es_module_from_code( - &deno_core::resolve_url("file:///eval.ts").map_err(error::to_anyhow)?, - source.to_string(), - ) - .await - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }) - .context("failed to load module")?; - - let main_override = script_entrypoint_override.unwrap_or("main".to_string()); - - // Inject parent trace context using enterSpan with a duck-typed span object. - // Uses job_id as trace_id so all spans are linked to the job. - // span_id is a placeholder - it gets overwritten by the OTLP handler with the real parent span_id. - #[cfg(all(feature = "private", feature = "enterprise"))] - let otel_context_inject = - if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - let trace_id = job_id.as_simple().to_string(); - format!( - r#"globalThis.__enterSpan?.({{ - isRecording: () => true, - spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) -}});"# - ) - } else { - String::new() - }; - - #[cfg(not(all(feature = "private", feature = "enterprise")))] - let otel_context_inject = ""; - - let script = js_runtime - .execute_script( - "", - format!( - r#" -function isAsyncIterable(obj) {{ - // return true; // TODO: remove this - return obj != null && typeof obj[Symbol.asyncIterator] === 'function'; -}} - -function processStreamIterative(res) {{ - const iterator = res[Symbol.asyncIterator](); - - function processLoop() {{ - return new Promise(function(resolve) {{ - function step() {{ - iterator.next().then(function(result) {{ - if (!result.done) {{ - const chunk = result.value; - console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); - // Continue the loop - step(); - }} else {{ - resolve("null"); - }} - }}).catch(function(error) {{ - resolve("null"); - }}); - }} - step(); - }}); - }} - - return processLoop(); -}} - -{otel_context_inject} - -let args = Deno.core.ops.op_get_static_args().map(JSON.parse) -import("file:///eval.ts").then((module) => module.{main_override}(...args)) - .then(res => {{ - if (isAsyncIterable(res)) {{ - return processStreamIterative(res) - }} else {{ - return JSON.stringify(res ?? null); - }} - }}) -"# - ), - ) - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }) - .context("native script initialization")?; - - let fut = js_runtime.resolve(script); - let global = js_runtime - .with_event_loop_promise(fut, PollEventLoopOptions::default()) - .await - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }); - - match global { - Ok(global) => { - let scope = &mut js_runtime.handle_scope(); - let local = v8::Local::new(scope, global); - // Deserialize a `v8` object into a Rust type using `serde_v8`, - // in this case deserialize to a JSON `Value`. - let r = serde_v8::from_v8::>(scope, local).map_err(error::to_anyhow)?; - Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) - } - Err(CoreError::Js(e)) => { - let stack_head = e.frames.first().and_then(|f| { - if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") { - Some(format!( - "{}\n", - source - .lines() - .nth((f.line_number.unwrap_or(1)) as usize - 1) - .unwrap_or("") - .to_string() - )) - } else { - None - } - }); - let stack_s = format!( - "{}{}", - stack_head.unwrap_or("".to_string()), - e.stack.unwrap_or("".to_string()) - ); - let stack = if stack_s.is_empty() { - None - } else { - Some(stack_s) - }; - Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({ - "message": e.message, - "stack": stack, - "name": e.name, - })))) - } - Err(e) => Err(Error::ExecutionErr(e.print_with_cause())), - } -} - -#[cfg(feature = "deno_core")] -#[op2] -#[serde] -fn op_get_static_args(op_state: Rc>) -> Vec> { - op_state - .borrow() - .borrow::() - .args - .iter() - .map(|x| x.as_ref().map(|y| y.get().to_string())) - .collect_vec() -} - -#[cfg(feature = "deno_core")] -#[op2(fast)] -fn op_log(op_state: Rc>, #[string] log: &str) { - // tracing::error!("log: |{}|", log); - if let Err(e) = op_state - .borrow_mut() - .borrow_mut::() - .s - .send(log.to_string()) - { - tracing::error!("failed to send log: {e}"); - } -} - -#[cfg(feature = "deno_core")] -#[cfg(test)] -mod tests { - - use serde_json::json; - use windmill_common::worker::to_raw_value; - - // Note this useful idiom: importing names from outer (for mod tests) scope. - use super::*; - - #[tokio::test] - async fn test_eval() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - env.insert( - "value".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - - let code = "value.test + params.test"; - - let ops = vec![op_get_context()]; - - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - - let options = RuntimeOptions { extensions: exts, ..Default::default() }; - - let mut runtime = JsRuntime::new(options); - { - let op_state = runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(TransformContext { flow_input: None, envs: env.clone() }) - } - - let res = eval( - &mut runtime, - code, - vec!["params".to_string(), "value".to_string()], - None, - false, - None, - ) - .await?; - assert_eq!(res.get(), "4"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_multiline() -> anyhow::Result<()> { - let env = vec![]; - let code = "let x = 5; -return `my ${x} -multiline template`"; - - let mut runtime = JsRuntime::new(RuntimeOptions::default()); - let res = eval(&mut runtime, code, env, None, false, None).await?; - assert_eq!(res.get(), "\"my 5\\nmultiline template\""); - Ok(()) - } - - #[tokio::test] - async fn test_eval_timeout() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - env.insert( - "value".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - - let code = r#"params.test"#; - - let mut js_runtime = JsRuntime::new(RuntimeOptions::default()); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(TransformContext { flow_input: None, envs: env.clone() }) - } - - let res = eval_timeout(code.to_string(), env, None, None, None, None, None).await?; - assert_eq!(res.get(), "2"); - Ok(()) - } - - // #[tokio::test] - // async fn test_eval_timeout_bug() -> anyhow::Result<()> { - // let ops = vec![op_get_static_args(), op_log()]; - // let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; - - // let deno_fetch_options = if let Some(cert_path) = env::var("DENO_CERT").ok() { - // let mut cert_store_provider = ContainerRootCertStoreProvider::new(); - // cert_store_provider.add_certificate(cert_path)?; - - // deno_fetch::Options { - // root_cert_store_provider: Some(Arc::new(cert_store_provider)), - // ..Default::default() - // } - // } else { - // Default::default() - // }; - - // let exts: Vec = vec![ - // deno_webidl::deno_webidl::init_ops(), - // deno_url::deno_url::init_ops(), - // deno_console::deno_console::init_ops(), - // deno_web::deno_web::init_ops::( - // Arc::new(BlobStore::default()), - // None, - // ), - // deno_fetch::deno_fetch::init_ops::(deno_fetch_options), - // deno_net::deno_net::init_ops::(None, None), - // ext, - // ]; - - // // Use our snapshot to provision our new runtime - // let options = RuntimeOptions { - // is_main: true, - // extensions: exts, - // create_params: Some( - // deno_core::v8::CreateParams::default() - // .heap_limits(0 as usize, 1024 * 1024 * 128 as usize), - // ), - // // startup_snapshot: None, - // startup_snapshot: Some(RUNTIME_SNAPSHOT), - // module_loader: Some(Rc::new(deno_core::FsModuleLoader)), - // extension_transpiler: None, - // ..Default::default() - // }; - - // let mut js_runtime: JsRuntime = JsRuntime::new(options); - // Ok(()) - // } - - // #[tokio::test] - // async fn test_eval_fetch_timeout() -> anyhow::Result<()> { - // let code = r#"export async function main() { return "" }"#; - - // let res = eval_fetch_timeout(code.to_string(), code.to_string(), None, Uuid::new_v4(), None, ).await?; - // assert_eq!(res.0.get(), "\"\""); - // Ok(()) - // } - - #[test] - fn test_should_add_return() { - // Simple expressions should get return added - assert_eq!(should_add_return("5"), true); - assert_eq!(should_add_return("x + y"), true); - assert_eq!(should_add_return("foo()"), true); - assert_eq!(should_add_return("obj.property"), true); - - // Object literals should get return added - assert_eq!(should_add_return("{ foo: 'bar' }"), true); - assert_eq!(should_add_return("{ a: 1, b: 2 }"), true); - assert_eq!(should_add_return("{}"), true); - - // Already has return - assert_eq!(should_add_return("return 5"), false); - assert_eq!(should_add_return("return x + y"), false); - assert_eq!(should_add_return("return;"), false); - assert_eq!(should_add_return("return"), false); - - // Should NOT add return for statements - assert_eq!(should_add_return("const x = 5"), false); - assert_eq!(should_add_return("let y = 10"), false); - assert_eq!(should_add_return("var z = 15"), false); - assert_eq!(should_add_return("if (x > 5) { return x; }"), false); - assert_eq!(should_add_return("for (let i = 0; i < 10; i++) {}"), false); - assert_eq!(should_add_return("while (true) {}"), false); - assert_eq!(should_add_return("function foo() {}"), false); - assert_eq!(should_add_return("throw new Error('test')"), false); - - // Multiple statements with semicolons (including block statements) - assert_eq!(should_add_return("let x = 5; x + 1"), false); - assert_eq!(should_add_return("{ const x = 5; return x; }"), false); - - // Edge case: "return" in a string should still get return prepended - assert_eq!(should_add_return("\"return this string\""), true); - assert_eq!(should_add_return("'return in single quotes'"), true); - assert_eq!(should_add_return("`return in template literal`"), true); - - // Semicolons in strings should not trigger multi-statement detection - assert_eq!(should_add_return("\"hello; world\""), true); - assert_eq!(should_add_return("'test; string'"), true); - assert_eq!(should_add_return("`template; literal`"), true); - } - - #[test] - fn test_contains_semicolon_outside_strings() { - // Semicolons outside strings - assert_eq!(contains_semicolon_outside_strings("let x = 5; x + 1"), true); - assert_eq!(contains_semicolon_outside_strings("x; y"), true); - - // Semicolons inside strings (should NOT be detected) - assert_eq!( - contains_semicolon_outside_strings("\"hello; world\""), - false - ); - assert_eq!(contains_semicolon_outside_strings("'test; string'"), false); - assert_eq!( - contains_semicolon_outside_strings("`template; literal`"), - false - ); - - // Mixed cases - assert_eq!( - contains_semicolon_outside_strings("let x = 'hello; world'; x"), - true - ); - assert_eq!( - contains_semicolon_outside_strings("console.log(\"test; string\")"), - false - ); - - // No semicolons - assert_eq!(contains_semicolon_outside_strings("x + y"), false); - assert_eq!(contains_semicolon_outside_strings("foo()"), false); - } -} diff --git a/backend/windmill-worker/src/js_eval_parity_tests.rs b/backend/windmill-worker/src/js_eval_parity_tests.rs deleted file mode 100644 index 0863598e76..0000000000 --- a/backend/windmill-worker/src/js_eval_parity_tests.rs +++ /dev/null @@ -1,4785 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -//! Feature parity tests for deno_core vs rquickjs expression evaluation. -//! -//! This module ensures both JavaScript engines produce identical results for -//! the same expressions, validating that QuickJS can be used as a drop-in -//! replacement for deno_core in flow expression evaluation. - -#[cfg(all(test, feature = "deno_core", feature = "quickjs"))] -mod parity_tests { - use std::collections::HashMap; - use std::sync::Arc; - - use serde_json::json; - use serde_json::value::RawValue; - use windmill_common::worker::to_raw_value; - - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - /// Helper to run the same test on both engines and compare results - async fn test_parity( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - ) -> anyhow::Result<()> { - test_parity_with_flow_env(expr, transform_context, flow_input, None).await - } - - /// Helper to run the same test on both engines with flow_env support - async fn test_parity_with_flow_env( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - ) -> anyhow::Result<()> { - let deno_result = eval_timeout( - expr.to_string(), - transform_context.clone(), - flow_input.clone(), - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let quickjs_result = eval_timeout_quickjs( - expr.to_string(), - transform_context, - flow_input, - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - // Parse both results to compare as JSON values (handles formatting differences) - let deno_value: serde_json::Value = serde_json::from_str(deno_result.get())?; - let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; - - assert_eq!( - deno_value, - quickjs_value, - "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, - deno_result.get(), - quickjs_result.get() - ); - - Ok(()) - } - - #[tokio::test] - async fn parity_simple_arithmetic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); - - test_parity("x + y", env.clone(), None).await?; - test_parity("x - y", env.clone(), None).await?; - test_parity("x * y", env.clone(), None).await?; - test_parity("x / y", env.clone(), None).await?; - test_parity("x % y", env.clone(), None).await?; - test_parity("x ** 2", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_property_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test", - "value": 42, - "nested": { - "deep": { - "property": "found" - } - } - }))), - ); - - test_parity("obj.name", env.clone(), None).await?; - test_parity("obj.value", env.clone(), None).await?; - test_parity("obj.nested.deep.property", env.clone(), None).await?; - test_parity("obj['name']", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - test_parity("arr.length", env.clone(), None).await?; - test_parity("arr[0]", env.clone(), None).await?; - test_parity("arr.map(x => x * 2)", env.clone(), None).await?; - test_parity("arr.filter(x => x > 2)", env.clone(), None).await?; - test_parity("arr.reduce((a, b) => a + b, 0)", env.clone(), None).await?; - test_parity("arr.find(x => x > 3)", env.clone(), None).await?; - test_parity("arr.some(x => x > 4)", env.clone(), None).await?; - test_parity("arr.every(x => x > 0)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_string_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "s".to_string(), - Arc::new(to_raw_value(&json!("Hello World"))), - ); - - test_parity("s.toLowerCase()", env.clone(), None).await?; - test_parity("s.toUpperCase()", env.clone(), None).await?; - test_parity("s.length", env.clone(), None).await?; - test_parity("s.split(' ')", env.clone(), None).await?; - test_parity("s.replace('World', 'QuickJS')", env.clone(), None).await?; - test_parity("s.includes('World')", env.clone(), None).await?; - test_parity("s.startsWith('Hello')", env.clone(), None).await?; - test_parity("s.trim()", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_ternary_and_conditionals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(10)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity("x > y ? 'bigger' : 'smaller'", env.clone(), None).await?; - test_parity("x === 10 ? true : false", env.clone(), None).await?; - test_parity("x > 5 && y < 10", env.clone(), None).await?; - test_parity("x > 20 || y < 10", env.clone(), None).await?; - test_parity("!false", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_creation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("name".to_string(), Arc::new(to_raw_value(&json!("test")))); - env.insert("value".to_string(), Arc::new(to_raw_value(&json!(42)))); - - test_parity("({ foo: 'bar' })", env.clone(), None).await?; - test_parity("({ name, value })", env.clone(), None).await?; - test_parity("({ ...{ a: 1 }, b: 2 })", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_null_undefined() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("null", env.clone(), None).await?; - test_parity("undefined", env.clone(), None).await?; - - let mut env_with_null = HashMap::new(); - env_with_null.insert("x".to_string(), Arc::new(to_raw_value(&json!(null)))); - test_parity("x", env_with_null.clone(), None).await?; - test_parity("x ?? 'default'", env_with_null.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input() -> anyhow::Result<()> { - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test_flow"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(100))); - flow_input.insert( - "config".to_string(), - to_raw_value(&json!({"enabled": true})), - ); - - let fi = Some(mappable_rc::Marc::new(flow_input)); - - test_parity("flow_input.name", HashMap::new(), fi.clone()).await?; - test_parity("flow_input.count", HashMap::new(), fi.clone()).await?; - test_parity("flow_input.config.enabled", HashMap::new(), fi.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_template_literals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("name".to_string(), Arc::new(to_raw_value(&json!("World")))); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity("`Hello ${name}!`", env.clone(), None).await?; - test_parity("`The answer is ${x * 2}`", env.clone(), None).await?; - test_parity("`Multi\nline`", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_json_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - test_parity("JSON.stringify(obj)", env.clone(), None).await?; - test_parity("Object.keys(obj)", env.clone(), None).await?; - test_parity("Object.values(obj)", env.clone(), None).await?; - // Note: Object.entries order might differ, so we skip that - - Ok(()) - } - - #[tokio::test] - async fn parity_math_operations() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("Math.max(1, 5, 3)", env.clone(), None).await?; - test_parity("Math.min(1, 5, 3)", env.clone(), None).await?; - test_parity("Math.abs(-5)", env.clone(), None).await?; - test_parity("Math.floor(3.7)", env.clone(), None).await?; - test_parity("Math.ceil(3.2)", env.clone(), None).await?; - test_parity("Math.round(3.5)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_type_coercion() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("str".to_string(), Arc::new(to_raw_value(&json!("123")))); - - test_parity("String(num)", env.clone(), None).await?; - test_parity("Number(str)", env.clone(), None).await?; - test_parity("Boolean(num)", env.clone(), None).await?; - test_parity("parseInt('42px')", env.clone(), None).await?; - test_parity("parseFloat('3.14')", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_spread() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr1".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3]))), - ); - env.insert( - "arr2".to_string(), - Arc::new(to_raw_value(&json!([4, 5, 6]))), - ); - - test_parity("[...arr1, ...arr2]", env.clone(), None).await?; - test_parity("[0, ...arr1, 99]", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_statements() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity( - r#"let y = x * 2; - return y + 1"#, - env.clone(), - None, - ) - .await?; - - test_parity( - r#"const result = x > 3 ? 'big' : 'small'; - return result"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_nullish() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": {"b": 1}}))), - ); - env.insert("empty".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining - test_parity("obj?.a?.b", env.clone(), None).await?; - test_parity("obj?.a?.c", env.clone(), None).await?; - test_parity("obj?.x?.y", env.clone(), None).await?; - test_parity("empty?.foo", env.clone(), None).await?; - - // Nullish coalescing - test_parity("null ?? 'default'", env.clone(), None).await?; - test_parity("undefined ?? 'default'", env.clone(), None).await?; - test_parity("0 ?? 'default'", env.clone(), None).await?; - test_parity("'' ?? 'default'", env.clone(), None).await?; - test_parity("false ?? 'default'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_destructuring() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"name": "test", "value": 42}))), - ); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Object destructuring - test_parity( - "const { name, value } = obj; return { name, value }", - env.clone(), - None, - ) - .await?; - - // Array destructuring - test_parity( - "const [first, second, ...rest] = arr; return { first, second, rest }", - env.clone(), - None, - ) - .await?; - - // Default values - test_parity( - "const { missing = 'default' } = obj; return missing", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_number_edge_cases() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Basic number operations - test_parity("Number.MAX_SAFE_INTEGER", env.clone(), None).await?; - test_parity("Number.MIN_SAFE_INTEGER", env.clone(), None).await?; - test_parity("Number.isInteger(5)", env.clone(), None).await?; - test_parity("Number.isInteger(5.5)", env.clone(), None).await?; - test_parity("Number.isFinite(Infinity)", env.clone(), None).await?; - test_parity("Number.isNaN(NaN)", env.clone(), None).await?; - - // Floating point - test_parity("0.1 + 0.2", env.clone(), None).await?; - test_parity("Math.round((0.1 + 0.2) * 10) / 10", env.clone(), None).await?; - - // Special values (these serialize to null in JSON) - test_parity("isNaN(NaN)", env.clone(), None).await?; - test_parity("isFinite(100)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_basic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello world 123"))), - ); - - // Basic regex operations - test_parity("/hello/.test(str)", env.clone(), None).await?; - test_parity("str.match(/\\d+/)?.[0]", env.clone(), None).await?; - test_parity("str.replace(/world/, 'universe')", env.clone(), None).await?; - test_parity("str.split(/\\s+/)", env.clone(), None).await?; - - // Global flag - test_parity("'aaa'.replace(/a/g, 'b')", env.clone(), None).await?; - - // Case insensitive - test_parity("/HELLO/i.test(str)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_unicode_strings() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "emoji".to_string(), - Arc::new(to_raw_value(&json!("Hello 👋 World 🌍"))), - ); - env.insert( - "chinese".to_string(), - Arc::new(to_raw_value(&json!("你好世界"))), - ); - env.insert( - "mixed".to_string(), - Arc::new(to_raw_value(&json!("Héllo Wörld"))), - ); - - // Basic operations on unicode strings - test_parity("emoji.includes('👋')", env.clone(), None).await?; - test_parity("chinese.length", env.clone(), None).await?; - test_parity("mixed.toUpperCase()", env.clone(), None).await?; - test_parity("mixed.toLowerCase()", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_basic() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Static Date methods (deterministic) - test_parity("Date.parse('2024-01-15T00:00:00.000Z')", env.clone(), None).await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCFullYear()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCMonth()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCDate()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').toISOString()", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_serialization() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Test direct Date object serialization (the key issue that was fixed) - // Both engines should serialize Date to ISO string via toJSON - test_parity("new Date('2024-01-15T12:30:00.000Z')", env.clone(), None).await?; - - // Date within an object - test_parity( - "({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })", - env.clone(), - None, - ) - .await?; - - // Date within an array - test_parity( - "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]", - env.clone(), - None, - ) - .await?; - - // Deeply nested Date - test_parity( - "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })", - env.clone(), - None, - ) - .await?; - - // Custom object with toJSON (arrow function style) - test_parity( - "({ value: 42, toJSON: () => ({ converted: 84 }) })", - env.clone(), - None, - ) - .await?; - - // toJSON that returns a Date (should be further serialized) - test_parity( - "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_object_serialization() -> anyhow::Result<()> { - let env = HashMap::new(); - - // RegExp serialization (both should return {}) - test_parity("/test/gi", env.clone(), None).await?; - - // Map serialization (both should return {}) - test_parity("new Map([['key', 'value']])", env.clone(), None).await?; - - // Set serialization (both should return {}) - test_parity("new Set([1, 2, 3])", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9, 2, 6]))), - ); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!([[1, 2], [3, 4], [5, 6]]))), - ); - - // Sorting (note: sort mutates, so we slice first) - test_parity("[...arr].sort((a, b) => a - b)", env.clone(), None).await?; - test_parity("[...arr].sort((a, b) => b - a)", env.clone(), None).await?; - - // Flat operations - test_parity("nested.flat()", env.clone(), None).await?; - test_parity("nested.flatMap(x => x)", env.clone(), None).await?; - - // indexOf, includes - test_parity("arr.indexOf(5)", env.clone(), None).await?; - test_parity("arr.indexOf(99)", env.clone(), None).await?; - test_parity("arr.includes(9)", env.clone(), None).await?; - - // slice, splice behavior - test_parity("arr.slice(2, 5)", env.clone(), None).await?; - test_parity("arr.slice(-3)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_logical_operators() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("a".to_string(), Arc::new(to_raw_value(&json!(true)))); - env.insert("b".to_string(), Arc::new(to_raw_value(&json!(false)))); - env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - // Short-circuit evaluation - test_parity("a && 'yes'", env.clone(), None).await?; - test_parity("b && 'yes'", env.clone(), None).await?; - test_parity("b || 'no'", env.clone(), None).await?; - test_parity("a || 'no'", env.clone(), None).await?; - - // Logical assignment (ES2021) - test_parity("let y = null; y ??= 10; return y", env.clone(), None).await?; - test_parity("let y = 5; y ??= 10; return y", env.clone(), None).await?; - - // Complex conditions - test_parity("(a && x > 3) || (b && x < 3)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_typeof_instanceof() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([1, 2, 3])))); - env.insert("obj".to_string(), Arc::new(to_raw_value(&json!({"a": 1})))); - env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // typeof - test_parity("typeof str", env.clone(), None).await?; - test_parity("typeof num", env.clone(), None).await?; - test_parity("typeof arr", env.clone(), None).await?; - test_parity("typeof obj", env.clone(), None).await?; - test_parity("typeof n", env.clone(), None).await?; - test_parity("typeof undefined", env.clone(), None).await?; - - // Array.isArray - test_parity("Array.isArray(arr)", env.clone(), None).await?; - test_parity("Array.isArray(obj)", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX MULTILINE EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_multiline_complex_logic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "users".to_string(), - Arc::new(to_raw_value(&json!([ - {"name": "Alice", "age": 30, "role": "admin"}, - {"name": "Bob", "age": 25, "role": "user"}, - {"name": "Charlie", "age": 35, "role": "admin"}, - {"name": "Diana", "age": 28, "role": "user"} - ]))), - ); - - // Complex filtering and mapping - test_parity( - r#" - const admins = users.filter(u => u.role === 'admin'); - const names = admins.map(u => u.name); - return names.join(', ') - "#, - env.clone(), - None, - ) - .await?; - - // Aggregation with reduce - test_parity( - r#" - const totalAge = users.reduce((sum, u) => sum + u.age, 0); - const avgAge = totalAge / users.length; - return Math.round(avgAge) - "#, - env.clone(), - None, - ) - .await?; - - // Group by operation - test_parity( - r#" - const grouped = users.reduce((acc, u) => { - if (!acc[u.role]) acc[u.role] = []; - acc[u.role].push(u.name); - return acc; - }, {}); - return grouped - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_data_transformation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [ - {"id": 1, "price": 100, "quantity": 2}, - {"id": 2, "price": 50, "quantity": 5}, - {"id": 3, "price": 75, "quantity": 3} - ], - "discount": 0.1 - }))), - ); - - // Calculate total with discount - test_parity( - r#" - const subtotals = data.items.map(item => item.price * item.quantity); - const total = subtotals.reduce((a, b) => a + b, 0); - const discounted = total * (1 - data.discount); - return { subtotals, total, discounted } - "#, - env.clone(), - None, - ) - .await?; - - // Transform data structure - test_parity( - r#" - const result = data.items.map(item => ({ - ...item, - subtotal: item.price * item.quantity, - discountedSubtotal: item.price * item.quantity * (1 - data.discount) - })); - return result - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_conditional_logic() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let mut env = HashMap::new(); - env.insert( - "status".to_string(), - Arc::new(to_raw_value(&json!("pending"))), - ); - env.insert("retries".to_string(), Arc::new(to_raw_value(&json!(3)))); - env.insert("maxRetries".to_string(), Arc::new(to_raw_value(&json!(5)))); - - // Complex conditional with multiple branches - test_parity( - r#" - let action; - if (status === 'success') { - action = 'complete'; - } else if (status === 'pending' && retries < maxRetries) { - action = 'retry'; - } else if (status === 'pending') { - action = 'fail'; - } else { - action = 'unknown'; - } - return { action, retriesLeft: maxRetries - retries } - "#, - env.clone(), - None, - ) - .await?; - - // Switch-like using object lookup - test_parity( - r#" - const actions = { - 'success': () => ({ next: 'complete', message: 'Done!' }), - 'pending': () => ({ next: 'retry', message: `Retry ${retries + 1}/${maxRetries}` }), - 'failed': () => ({ next: 'stop', message: 'Giving up' }) - }; - const handler = actions[status] || (() => ({ next: 'fallback', message: 'Unknown status' })); - return handler() - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // ARROW FUNCTION VARIATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_arrow_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "numbers".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Concise body (implicit return) - test_parity("numbers.map(n => n * 2)", env.clone(), None).await?; - - // Block body (explicit return) - test_parity("numbers.map(n => { return n * 2; })", env.clone(), None).await?; - - // Multiple parameters - test_parity("numbers.reduce((acc, n) => acc + n, 0)", env.clone(), None).await?; - - // Destructuring in parameters - test_parity( - r#" - const pairs = [[1, 2], [3, 4], [5, 6]]; - return pairs.map(([a, b]) => a + b) - "#, - HashMap::new(), - None, - ) - .await?; - - // Object destructuring in parameters - test_parity( - r#" - const items = [{x: 1, y: 2}, {x: 3, y: 4}]; - return items.map(({x, y}) => x * y) - "#, - HashMap::new(), - None, - ) - .await?; - - // Nested arrow functions - test_parity( - "numbers.map(n => numbers.filter(m => m !== n))", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // TRY-CATCH EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_try_catch() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let env = HashMap::new(); - - // Basic try-catch - test_parity( - r#" - try { - return JSON.parse('{"valid": true}'); - } catch (e) { - return { problem: e.message }; - } - "#, - env.clone(), - None, - ) - .await?; - - // Try-catch with invalid JSON - test_parity( - r#" - try { - return JSON.parse('invalid json'); - } catch (e) { - return { problem: 'parse_failed' }; - } - "#, - env.clone(), - None, - ) - .await?; - - // Try-catch-finally - test_parity( - r#" - let result = 'initial'; - try { - result = 'try'; - } catch (e) { - result = 'catch'; - } finally { - result = result + '_finally'; - } - return result - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX OBJECT OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_object_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "config".to_string(), - Arc::new(to_raw_value(&json!({ - "server": {"host": "localhost", "port": 8080}, - "database": {"host": "db.local", "port": 5432}, - "features": ["auth", "logging", "cache"] - }))), - ); - - // Object.assign - test_parity( - "Object.assign({}, config.server, { secure: true })", - env.clone(), - None, - ) - .await?; - - // Object spread with override - test_parity( - "({ ...config.server, port: 443, secure: true })", - env.clone(), - None, - ) - .await?; - - // Object.entries and Object.fromEntries - test_parity( - r#" - const entries = Object.entries(config.server); - const reversed = entries.map(([k, v]) => [k.toUpperCase(), v]); - return Object.fromEntries(reversed) - "#, - env.clone(), - None, - ) - .await?; - - // Deep clone pattern - test_parity("JSON.parse(JSON.stringify(config))", env.clone(), None).await?; - - // Computed property names - test_parity( - r#" - const key = 'dynamic'; - return { [key]: 'value', [`${key}_2`]: 'value2' } - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // STRING MANIPULATION ADVANCED - // ========================================================================= - - #[tokio::test] - async fn parity_string_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!(" Hello, World! "))), - ); - env.insert( - "path".to_string(), - Arc::new(to_raw_value(&json!("/api/v1/users/123/profile"))), - ); - - // Trim variants - test_parity("text.trim()", env.clone(), None).await?; - test_parity("text.trimStart()", env.clone(), None).await?; - test_parity("text.trimEnd()", env.clone(), None).await?; - - // Padding - test_parity("'42'.padStart(5, '0')", env.clone(), None).await?; - test_parity("'42'.padEnd(5, '-')", env.clone(), None).await?; - - // Repeat - test_parity("'ab'.repeat(3)", env.clone(), None).await?; - - // Path manipulation - test_parity( - "path.split('/').filter(p => p.length > 0)", - env.clone(), - None, - ) - .await?; - - // Template literal with expressions - test_parity( - r#"`Path parts: ${path.split('/').filter(p => p).length}`"#, - env.clone(), - None, - ) - .await?; - - // String search methods - test_parity("path.indexOf('/users/')", env.clone(), None).await?; - test_parity("path.lastIndexOf('/')", env.clone(), None).await?; - test_parity("path.substring(0, 7)", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // ARRAY MANIPULATION ADVANCED - // ========================================================================= - - #[tokio::test] - async fn parity_array_manipulation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!([ - {"id": 1, "name": "Apple", "category": "fruit"}, - {"id": 2, "name": "Carrot", "category": "vegetable"}, - {"id": 3, "name": "Banana", "category": "fruit"}, - {"id": 4, "name": "Broccoli", "category": "vegetable"} - ]))), - ); - - // find and findIndex - test_parity("items.find(i => i.name === 'Banana')", env.clone(), None).await?; - - test_parity( - "items.findIndex(i => i.name === 'Banana')", - env.clone(), - None, - ) - .await?; - - // Filter and sort chain - test_parity( - "items.filter(i => i.category === 'fruit').map(i => i.name).sort()", - env.clone(), - None, - ) - .await?; - - // Array.from with map function - test_parity( - "Array.from({length: 5}, (_, i) => i * 2)", - env.clone(), - None, - ) - .await?; - - // Array fill - test_parity("Array(3).fill(0)", env.clone(), None).await?; - - // Reverse (on copy to avoid mutation) - test_parity("[...items].reverse().map(i => i.name)", env.clone(), None).await?; - - // concat - test_parity("[1, 2].concat([3, 4], [5, 6])", env.clone(), None).await?; - - // join variations - test_parity("items.map(i => i.name).join(' | ')", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // REAL-WORLD FLOW EXPRESSION PATTERNS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_patterns_api_response() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "status": 200, - "data": { - "users": [ - {"id": 1, "email": "alice@example.com", "active": true}, - {"id": 2, "email": "bob@example.com", "active": false}, - {"id": 3, "email": "charlie@example.com", "active": true} - ], - "pagination": {"page": 1, "total": 50, "per_page": 10} - } - }))), - ); - - // Extract active users' emails - test_parity( - "previous_result.data.users.filter(u => u.active).map(u => u.email)", - env.clone(), - None, - ) - .await?; - - // Check if more pages exist - test_parity( - r#" - const { page, total, per_page } = previous_result.data.pagination; - return page * per_page < total - "#, - env.clone(), - None, - ) - .await?; - - // Transform to different structure - test_parity( - r#"({ - emails: previous_result.data.users.map(u => u.email), - activeCount: previous_result.data.users.filter(u => u.active).length, - hasMore: previous_result.data.pagination.page * previous_result.data.pagination.per_page < previous_result.data.pagination.total - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_failure_handling() -> anyhow::Result<()> { - // NOTE: We avoid using the literal word "error" in expressions because - // it triggers special error-handling code that has a bug with duplicate declarations. - - // Test with failure info in previous_result - let mut env_failure = HashMap::new(); - env_failure.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "failure": { - "name": "APIFailure", - "message": "Rate limit exceeded", - "code": 429 - } - }))), - ); - - // Check for failure presence - test_parity( - "previous_result?.failure ? true : false", - env_failure.clone(), - None, - ) - .await?; - - // Extract failure details - test_parity( - "previous_result.failure?.code ?? 500", - env_failure.clone(), - None, - ) - .await?; - - // Test with successful result (no failure) - let mut env_success = HashMap::new(); - env_success.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "data": "success" - }))), - ); - - test_parity( - "previous_result?.failure ? 'failed' : 'ok'", - env_success.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_conditional_branching() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "step_a".to_string(), - Arc::new(to_raw_value(&json!({"count": 5}))), - ); - env.insert( - "step_b".to_string(), - Arc::new(to_raw_value(&json!({"count": 10}))), - ); - env.insert("threshold".to_string(), Arc::new(to_raw_value(&json!(7)))); - - // Branch selection based on condition - test_parity( - "step_a.count > threshold ? 'high' : step_b.count > threshold ? 'medium' : 'low'", - env.clone(), - None, - ) - .await?; - - // Aggregate from multiple steps - test_parity( - "({ total: step_a.count + step_b.count, average: (step_a.count + step_b.count) / 2 })", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_data_mapping() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "source".to_string(), - Arc::new(to_raw_value(&json!({ - "firstName": "John", - "lastName": "Doe", - "birthDate": "1990-05-15", - "addresses": [ - {"type": "home", "city": "New York"}, - {"type": "work", "city": "Boston"} - ] - }))), - ); - - // Map to different schema - test_parity( - r#"({ - fullName: `${source.firstName} ${source.lastName}`, - birth_date: source.birthDate, - primary_city: source.addresses.find(a => a.type === 'home')?.city ?? source.addresses[0]?.city ?? 'Unknown', - all_cities: source.addresses.map(a => a.city) - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASES AND SPECIAL VALUES - // ========================================================================= - - #[tokio::test] - async fn parity_edge_cases_empty_values() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("emptyArray".to_string(), Arc::new(to_raw_value(&json!([])))); - env.insert( - "emptyObject".to_string(), - Arc::new(to_raw_value(&json!({}))), - ); - env.insert( - "emptyString".to_string(), - Arc::new(to_raw_value(&json!(""))), - ); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - - // Operations on empty values - test_parity("emptyArray.length", env.clone(), None).await?; - test_parity("emptyArray.map(x => x * 2)", env.clone(), None).await?; - test_parity("emptyArray.filter(x => x > 0)", env.clone(), None).await?; - test_parity("emptyArray.reduce((a, b) => a + b, 100)", env.clone(), None).await?; - - test_parity("Object.keys(emptyObject)", env.clone(), None).await?; - test_parity("Object.values(emptyObject)", env.clone(), None).await?; - - test_parity("emptyString.length", env.clone(), None).await?; - test_parity("emptyString || 'default'", env.clone(), None).await?; - test_parity("emptyString ?? 'default'", env.clone(), None).await?; - - test_parity("zero || 'default'", env.clone(), None).await?; - test_parity("zero ?? 'default'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_nested_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "deep".to_string(), - Arc::new(to_raw_value(&json!({ - "a": {"b": {"c": {"d": {"e": "found!"}}}} - }))), - ); - - // Deep property access - test_parity("deep.a.b.c.d.e", env.clone(), None).await?; - test_parity("deep?.a?.b?.c?.d?.e", env.clone(), None).await?; - test_parity("deep?.a?.b?.x?.y?.z", env.clone(), None).await?; - test_parity("deep?.a?.b?.x?.y?.z ?? 'not found'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_special_characters() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "key-with-dash": "value1", - "key.with.dots": "value2", - "key with spaces": "value3" - }))), - ); - - // Bracket notation for special keys - test_parity("data['key-with-dash']", env.clone(), None).await?; - test_parity("data['key.with.dots']", env.clone(), None).await?; - test_parity("data['key with spaces']", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_large_numbers() -> anyhow::Result<()> { - let mut env = HashMap::new(); - // Large but safe integers - env.insert( - "bigNum".to_string(), - Arc::new(to_raw_value(&json!(9007199254740991_i64))), // MAX_SAFE_INTEGER - ); - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // 2024-01-01 UTC - ); - - test_parity("bigNum", env.clone(), None).await?; - test_parity("timestamp", env.clone(), None).await?; - test_parity("new Date(timestamp).toISOString()", env.clone(), None).await?; - - // Arithmetic on large numbers - test_parity("bigNum - 1", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_boolean_coercion() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Falsy values - test_parity("Boolean(0)", env.clone(), None).await?; - test_parity("Boolean('')", env.clone(), None).await?; - test_parity("Boolean(null)", env.clone(), None).await?; - test_parity("Boolean(undefined)", env.clone(), None).await?; - test_parity("Boolean(NaN)", env.clone(), None).await?; - - // Truthy values - test_parity("Boolean(1)", env.clone(), None).await?; - test_parity("Boolean('hello')", env.clone(), None).await?; - test_parity("Boolean([])", env.clone(), None).await?; - test_parity("Boolean({})", env.clone(), None).await?; - - // Double negation coercion - test_parity("!!0", env.clone(), None).await?; - test_parity("!!1", env.clone(), None).await?; - test_parity("!!''", env.clone(), None).await?; - test_parity("!!'hello'", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // PROMISES AND ASYNC PATTERNS (without client) - // ========================================================================= - - #[tokio::test] - async fn parity_promise_resolve() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Basic Promise.resolve - test_parity("Promise.resolve(42)", env.clone(), None).await?; - - test_parity("Promise.resolve({ key: 'value' })", env.clone(), None).await?; - - // Promise.all with resolved values - test_parity( - "Promise.all([Promise.resolve(1), Promise.resolve(2), Promise.resolve(3)])", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // SET AND MAP OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_set_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3, 4]))), - ); - - // Deduplicate using Set - test_parity("[...new Set(arr)]", env.clone(), None).await?; - - // Set size - test_parity("new Set(arr).size", env.clone(), None).await?; - - // Set.has - test_parity("new Set(arr).has(3)", env.clone(), None).await?; - - test_parity("new Set(arr).has(99)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_map_operations() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Create Map and convert to object - test_parity( - r#" - const map = new Map([['a', 1], ['b', 2], ['c', 3]]); - return Object.fromEntries(map) - "#, - env.clone(), - None, - ) - .await?; - - // Map operations - test_parity( - r#" - const map = new Map(); - map.set('key1', 'value1'); - map.set('key2', 'value2'); - return map.get('key1') - "#, - env.clone(), - None, - ) - .await?; - - test_parity( - r#" - const map = new Map([['a', 1], ['b', 2]]); - return map.size - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // COMPARISON OPERATORS - // ========================================================================= - - #[tokio::test] - async fn parity_comparisons() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Strict equality - test_parity("1 === 1", env.clone(), None).await?; - test_parity("1 === '1'", env.clone(), None).await?; - test_parity("null === undefined", env.clone(), None).await?; - test_parity("null === null", env.clone(), None).await?; - - // Loose equality - test_parity("1 == '1'", env.clone(), None).await?; - test_parity("null == undefined", env.clone(), None).await?; - test_parity("0 == false", env.clone(), None).await?; - test_parity("'' == false", env.clone(), None).await?; - - // Inequality - test_parity("5 !== '5'", env.clone(), None).await?; - test_parity("5 != '5'", env.clone(), None).await?; - - // Comparison operators - test_parity("5 > 3", env.clone(), None).await?; - test_parity("5 >= 5", env.clone(), None).await?; - test_parity("3 < 5", env.clone(), None).await?; - test_parity("5 <= 5", env.clone(), None).await?; - - // String comparison - test_parity("'apple' < 'banana'", env.clone(), None).await?; - test_parity("'10' < '9'", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // BITWISE OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_bitwise() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("5 & 3", env.clone(), None).await?; - test_parity("5 | 3", env.clone(), None).await?; - test_parity("5 ^ 3", env.clone(), None).await?; - test_parity("~5", env.clone(), None).await?; - test_parity("5 << 2", env.clone(), None).await?; - test_parity("20 >> 2", env.clone(), None).await?; - test_parity("-5 >>> 0", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX REAL-WORLD SCENARIOS - // ========================================================================= - - #[tokio::test] - async fn parity_scenario_batch_processing() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let mut env = HashMap::new(); - env.insert( - "jobs".to_string(), - Arc::new(to_raw_value(&json!([ - {"id": 1, "status": "completed", "result": 100}, - {"id": 2, "status": "failed", "reason": "timeout"}, - {"id": 3, "status": "completed", "result": 200}, - {"id": 4, "status": "failed", "reason": "connection"}, - {"id": 5, "status": "completed", "result": 150} - ]))), - ); - - // Aggregate batch results - test_parity( - r#" - const completed = jobs.filter(j => j.status === 'completed'); - const failed = jobs.filter(j => j.status === 'failed'); - const totalResult = completed.reduce((sum, j) => sum + j.result, 0); - return { - totalJobs: jobs.length, - completedCount: completed.length, - failedCount: failed.length, - successRate: completed.length / jobs.length, - totalResult, - failureReasons: failed.map(j => j.reason) - } - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_webhook_payload() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "webhook".to_string(), - Arc::new(to_raw_value(&json!({ - "event": "user.created", - "timestamp": "2024-01-15T10:30:00Z", - "data": { - "user": { - "id": "usr_123", - "email": "newuser@example.com", - "metadata": { - "source": "signup", - "campaign": "winter_2024" - } - } - } - }))), - ); - - // Extract and transform webhook data - test_parity( - r#"({ - eventType: webhook.event.split('.')[1], - userId: webhook.data.user.id, - userEmail: webhook.data.user.email, - source: webhook.data.user.metadata?.source ?? 'unknown', - campaign: webhook.data.user.metadata?.campaign, - processedAt: new Date().toISOString().split('T')[0] - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_config_merge() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "defaults".to_string(), - Arc::new(to_raw_value(&json!({ - "timeout": 5000, - "retries": 3, - "headers": {"Content-Type": "application/json"}, - "features": {"logging": true, "caching": false} - }))), - ); - env.insert( - "overrides".to_string(), - Arc::new(to_raw_value(&json!({ - "timeout": 10000, - "headers": {"Authorization": "Bearer token"}, - "features": {"caching": true} - }))), - ); - - // Deep merge configuration - test_parity( - r#"({ - ...defaults, - ...overrides, - headers: { ...defaults.headers, ...overrides.headers }, - features: { ...defaults.features, ...overrides.features } - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } -} - -#[cfg(test)] -mod benchmark_tests { - use std::collections::HashMap; - use std::sync::Arc; - use std::time::Instant; - - use serde_json::json; - use windmill_common::worker::to_raw_value; - - /// Benchmark QuickJS expression evaluation startup time - #[cfg(feature = "quickjs")] - #[tokio::test] - async fn benchmark_quickjs_startup() -> anyhow::Result<()> { - use crate::js_eval_quickjs::eval_timeout_quickjs; - - let iterations = 100; - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout_quickjs( - "x + 1".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let duration = start.elapsed(); - - println!( - "QuickJS: {} iterations in {:?} ({:?} per iteration)", - iterations, - duration, - duration / iterations - ); - - Ok(()) - } - - /// Benchmark deno_core expression evaluation startup time - #[cfg(feature = "deno_core")] - #[tokio::test] - async fn benchmark_deno_startup() -> anyhow::Result<()> { - use crate::js_eval::eval_timeout; - - let iterations = 100; - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout( - "x + 1".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let duration = start.elapsed(); - - println!( - "deno_core: {} iterations in {:?} ({:?} per iteration)", - iterations, - duration, - duration / iterations - ); - - Ok(()) - } - - /// Benchmark both engines with a complex expression - #[cfg(all(feature = "deno_core", feature = "quickjs"))] - #[tokio::test] - async fn benchmark_complex_expression() -> anyhow::Result<()> { - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - let iterations = 50; - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "multiplier": 2 - }))), - ); - - let expr = "data.items.filter(x => x > 3).map(x => x * data.multiplier).reduce((a, b) => a + b, 0)"; - - // QuickJS - let start = Instant::now(); - for _ in 0..iterations { - let _ = - eval_timeout_quickjs(expr.to_string(), env.clone(), None, None, None, None, None) - .await?; - } - let quickjs_duration = start.elapsed(); - - // deno_core - let start = Instant::now(); - for _ in 0..iterations { - let _ = - eval_timeout(expr.to_string(), env.clone(), None, None, None, None, None).await?; - } - let deno_duration = start.elapsed(); - - println!( - "Complex expression benchmark ({} iterations):\n QuickJS: {:?} ({:?}/iter)\n deno_core: {:?} ({:?}/iter)\n Speedup: {:.2}x", - iterations, - quickjs_duration, quickjs_duration / iterations, - deno_duration, deno_duration / iterations, - deno_duration.as_secs_f64() / quickjs_duration.as_secs_f64() - ); - - Ok(()) - } -} - -/// Comprehensive flow simulation parity tests -/// Tests expression evaluation in contexts that simulate real flow execution -#[cfg(all(test, feature = "deno_core", feature = "quickjs"))] -mod flow_simulation_parity_tests { - use std::collections::HashMap; - use std::sync::Arc; - - use serde_json::json; - use serde_json::value::RawValue; - use windmill_common::worker::to_raw_value; - - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - /// Helper to run the same test on both engines and compare results - async fn test_parity( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - ) -> anyhow::Result<()> { - let deno_result = eval_timeout( - expr.to_string(), - transform_context.clone(), - flow_input.clone(), - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let quickjs_result = eval_timeout_quickjs( - expr.to_string(), - transform_context, - flow_input, - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let deno_value: serde_json::Value = serde_json::from_str(deno_result.get())?; - let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; - - assert_eq!( - deno_value, - quickjs_value, - "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, - deno_result.get(), - quickjs_result.get() - ); - - Ok(()) - } - - // ========================================================================= - // SIMULATED FLOW CONTEXT: Multi-step flow with various step results - // ========================================================================= - - fn create_multi_step_flow_context() -> ( - HashMap>>, - Option>>>, - Option>>, - ) { - let mut transform_context = HashMap::new(); - - // Step 'a' result: simple number - transform_context.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); - - // Step 'b' result: object with nested data - transform_context.insert( - "b".to_string(), - Arc::new(to_raw_value(&json!({ - "status": "success", - "data": { - "users": [ - {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, - {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, - {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} - ], - "total": 3, - "metadata": { - "page": 1, - "hasMore": true - } - } - }))), - ); - - // Step 'c' result: array of numbers (from a for-loop) - transform_context.insert( - "c".to_string(), - Arc::new(to_raw_value(&json!([10, 20, 30, 40, 50]))), - ); - - // Step 'd' result: null (simulating a step that returned null) - transform_context.insert("d".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Step 'e' result: error object (simulating a failed step with continue_on_error) - transform_context.insert( - "e".to_string(), - Arc::new(to_raw_value(&json!({ - "error": { - "name": "ValidationError", - "message": "Invalid input provided", - "step_id": "e" - } - }))), - ); - - // Step 'f' result: deeply nested object - transform_context.insert( - "f".to_string(), - Arc::new(to_raw_value(&json!({ - "level1": { - "level2": { - "level3": { - "level4": { - "value": "deeply_nested" - } - } - } - } - }))), - ); - - // Step 'g' result: array of mixed types - transform_context.insert( - "g".to_string(), - Arc::new(to_raw_value(&json!([ - "string", - 123, - true, - null, - {"key": "value"}, - [1, 2, 3] - ]))), - ); - - // previous_result (the last executed step, 'g') - transform_context.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!([ - "string", - 123, - true, - null, - {"key": "value"}, - [1, 2, 3] - ]))), - ); - - // flow_input - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test_flow"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(100))); - flow_input.insert("enabled".to_string(), to_raw_value(&json!(true))); - flow_input.insert( - "config".to_string(), - to_raw_value(&json!({ - "timeout": 30, - "retries": 3, - "options": ["fast", "secure"] - })), - ); - flow_input.insert( - "items".to_string(), - to_raw_value(&json!([ - {"id": 1, "value": "first"}, - {"id": 2, "value": "second"}, - {"id": 3, "value": "third"} - ])), - ); - - // flow_env - let mut flow_env = HashMap::new(); - flow_env.insert("ENV".to_string(), to_raw_value(&json!("production"))); - flow_env.insert("DEBUG".to_string(), to_raw_value(&json!(false))); - flow_env.insert("VERSION".to_string(), to_raw_value(&json!("1.2.3"))); - - ( - transform_context, - Some(mappable_rc::Marc::new(flow_input)), - Some(flow_env), - ) - } - - // ========================================================================= - // INPUT TRANSFORM EXPRESSIONS (step inputs) - // ========================================================================= - - #[tokio::test] - async fn parity_input_transform_direct_reference() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Direct step result reference - test_parity("a", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_property_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Nested property access - test_parity("b.status", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users[0].name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users[1].roles", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "b.data.metadata.hasMore", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Deeply nested - test_parity( - "f.level1.level2.level3.level4.value", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_array_operations() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Array indexing - test_parity("c[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c[c.length - 1]", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Array methods - test_parity("c.map(x => x * 2)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.filter(x => x > 25)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "c.reduce((acc, x) => acc + x, 0)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity("c.find(x => x === 30)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.some(x => x > 40)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.every(x => x > 0)", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Chained operations - test_parity( - "c.filter(x => x > 20).map(x => x / 10)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_complex_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Complex data extraction from step 'b' - test_parity( - "b.data.users.filter(u => u.active).map(u => u.name)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - test_parity( - "b.data.users.filter(u => u.roles.includes('admin'))[0]?.name", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - test_parity( - "b.data.users.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Combining multiple step results - test_parity("a + c[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("a * b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - // ========================================================================= - // FLOW_INPUT EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_input_simple() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_input.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.count", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.enabled", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_nested() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity( - "flow_input.config.timeout", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.config.retries", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.config.options", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.config.options[0]", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_array_operations() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity( - "flow_input.items.length", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.items[0].id", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.items.map(i => i.value)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input.items.find(i => i.id === 2)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_combined_with_steps() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Combining flow_input with step results - test_parity("flow_input.count + a", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "flow_input.config.timeout * b.data.total", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Conditional based on flow_input - test_parity( - "flow_input.enabled ? b.data.users : []", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // FLOW_ENV EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_env_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_env.ENV", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_env.DEBUG", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_env.VERSION", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_env_conditionals() -> anyhow::Result<()> { - // Test flow_env conditionals with explicit flow_env reference in context - let mut ctx = HashMap::new(); - ctx.insert( - "env_val".to_string(), - Arc::new(to_raw_value(&json!("production"))), - ); - ctx.insert( - "debug_val".to_string(), - Arc::new(to_raw_value(&json!(false))), - ); - - test_parity( - "env_val === 'production' ? 'prod' : 'dev'", - ctx.clone(), - None, - None, - ) - .await?; - - test_parity( - "debug_val ? 'debug mode' : 'normal'", - ctx.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // ITERATOR EXPRESSIONS (for forloopflow) - // ========================================================================= - - #[tokio::test] - async fn parity_iterator_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Typical iterator expressions - test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; // Direct array - test_parity("b.data.users", ctx.clone(), fi.clone(), fe.clone()).await?; // Nested array - test_parity("flow_input.items", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Transformed iterators - test_parity( - "c.map(x => ({value: x, doubled: x * 2}))", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "b.data.users.filter(u => u.active)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Range-like iteration - test_parity( - "Array.from({length: 5}, (_, i) => i)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_forloop_inner_expressions() -> anyhow::Result<()> { - // Simulate expressions inside a for-loop where flow_input.iter exists - let mut ctx = HashMap::new(); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({"value": 42, "index": 2}))), - ); - - let mut flow_input = HashMap::new(); - flow_input.insert( - "iter".to_string(), - to_raw_value(&json!({ - "index": 2, - "value": {"id": 3, "name": "test_item"} - })), - ); - flow_input.insert("name".to_string(), to_raw_value(&json!("parent_flow"))); - - let fi = Some(mappable_rc::Marc::new(flow_input)); - - test_parity("flow_input.iter.index", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value.id", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value.name", ctx.clone(), fi.clone(), None).await?; - - // Combining iter with other flow_input - test_parity( - "`Item ${flow_input.iter.index} of ${flow_input.name}`", - ctx.clone(), - fi.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // BRANCH CONDITION EXPRESSIONS (for branchone) - // ========================================================================= - - #[tokio::test] - async fn parity_branch_conditions() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - // Simple boolean conditions - test_parity("a > 40", ctx.clone(), fi.clone(), None).await?; - test_parity("b.status === 'success'", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.enabled", ctx.clone(), fi.clone(), None).await?; - - // Complex boolean conditions - test_parity( - "a > 40 && b.status === 'success'", - ctx.clone(), - fi.clone(), - None, - ) - .await?; - test_parity("a < 50 || b.data.total > 5", ctx.clone(), fi.clone(), None).await?; - - // Conditions with array checks - test_parity("b.data.users.length > 0", ctx.clone(), fi.clone(), None).await?; - test_parity( - "b.data.users.some(u => u.active)", - ctx.clone(), - fi.clone(), - None, - ) - .await?; - test_parity("c.includes(30)", ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // SKIP_IF / STOP_AFTER_IF EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_skip_if_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Skip based on previous result - test_parity( - "previous_result === null", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "previous_result.length === 0", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Skip based on flow_input - test_parity("!flow_input.enabled", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "flow_input.count === 0", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Skip based on step result - test_parity("d === null", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "e?.error !== undefined", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_stop_after_if_expressions() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - // Stop conditions (avoid previous_result?.error pattern which has issues with error extraction) - test_parity("a >= 42", ctx.clone(), fi.clone(), None).await?; - test_parity( - "b.data.metadata.hasMore === false", - ctx.clone(), - fi.clone(), - None, - ) - .await?; - test_parity("b.status !== 'success'", ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // UNDEFINED/MISSING STEP RESULTS (simulating non-executed branches) - // ========================================================================= - - #[tokio::test] - async fn parity_missing_step_with_optional_chaining() -> anyhow::Result<()> { - // Context where some steps weren't executed (e.g., branch not taken) - let mut ctx = HashMap::new(); - ctx.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); - // 'b' was never executed (branch not taken) - ctx.insert("c".to_string(), Arc::new(to_raw_value(&json!(null)))); // Step returned null - - // Safe access to potentially missing step - test_parity("a", ctx.clone(), None, None).await?; - test_parity("c", ctx.clone(), None, None).await?; - - // Optional chaining on null - test_parity("c?.value", ctx.clone(), None, None).await?; - test_parity("c?.nested?.deep", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_null_coalescing_for_missing_data() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Nullish coalescing - test_parity("d ?? 'default'", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "d?.value ?? 'not found'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // With nested access - test_parity( - "b.data.missing?.value ?? 'fallback'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // ERROR HANDLING EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_error_object_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Accessing error from step 'e' - test_parity("e.error.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e.error.message", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e.error.step_id", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Conditional based on error - test_parity( - "e.error ? `Error: ${e.error.message}` : 'OK'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_variable_extraction() -> anyhow::Result<()> { - // Simulate previous_result being an error - let mut ctx = HashMap::new(); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "error": { - "name": "RuntimeError", - "message": "Something went wrong" - } - }))), - ); - - // The 'error' variable is extracted from previous_result - test_parity("error.name", ctx.clone(), None, None).await?; - test_parity("error.message", ctx.clone(), None, None).await?; - test_parity("`${error.name}: ${error.message}`", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_from_parallel_results() -> anyhow::Result<()> { - // Simulate previous_result being an array with errors (from parallel branches) - let mut ctx = HashMap::new(); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!([ - {"result": "success"}, - {"error": {"name": "Error1", "message": "First error", "step_id": "branch_1"}}, - {"result": "also success"}, - {"error": {"name": "Error2", "message": "Second error", "step_id": "branch_2"}} - ]))), - ); - - // Access the aggregated error - test_parity("error.name", ctx.clone(), None, None).await?; - test_parity("error.message", ctx.clone(), None, None).await?; - test_parity("error.errors", ctx.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // OBJECT CONSTRUCTION EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_object_construction() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Building objects from step results - test_parity( - "({ count: a, users: b.data.users })", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - test_parity( - "({ ...flow_input.config, extra: 'value' })", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Computed properties - test_parity( - "({ [`step_${a}`]: b.status })", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_construction() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Spread operator - test_parity("[...c, 60, 70]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("[a, ...c.slice(0, 2)]", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Array from step results - test_parity( - "[b.data.users[0], b.data.users[2]]", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // MULTILINE / COMPLEX EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_multiline_data_processing() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity( - r#" - let users = b.data.users; - let activeUsers = users.filter(u => u.active); - let adminUsers = activeUsers.filter(u => u.roles.includes('admin')); - return adminUsers.map(u => u.name); - "#, - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_conditional_logic() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - test_parity( - r#" - if (flow_input.enabled) { - return { mode: 'enabled', data: b.data.users.filter(u => u.active) }; - } else { - return { mode: 'disabled', data: b.data.users }; - } - "#, - ctx.clone(), - fi.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_aggregation() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - test_parity( - r#" - const summary = { - stepA: a, - stepB_status: b.status, - stepC_sum: c.reduce((acc, x) => acc + x, 0), - stepC_count: c.length, - activeUserCount: b.data.users.filter(u => u.active).length, - flowName: flow_input.name, - enabled: flow_input.enabled - }; - return summary; - "#, - ctx.clone(), - fi.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASES - // ========================================================================= - - #[tokio::test] - async fn parity_empty_arrays_and_objects() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert("emptyArr".to_string(), Arc::new(to_raw_value(&json!([])))); - ctx.insert("emptyObj".to_string(), Arc::new(to_raw_value(&json!({})))); - - test_parity("emptyArr.length", ctx.clone(), None, None).await?; - test_parity("emptyArr.map(x => x)", ctx.clone(), None, None).await?; - test_parity("emptyArr.filter(x => true)", ctx.clone(), None, None).await?; - test_parity("Object.keys(emptyObj)", ctx.clone(), None, None).await?; - test_parity("Object.values(emptyObj)", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_large_numbers() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert( - "bigInt".to_string(), - Arc::new(to_raw_value(&json!(9007199254740991_i64))), - ); // MAX_SAFE_INTEGER - ctx.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1703980800000_i64))), - ); // Typical timestamp - - test_parity("bigInt", ctx.clone(), None, None).await?; - test_parity("timestamp", ctx.clone(), None, None).await?; - test_parity("bigInt + 1", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_characters_in_strings() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "message": "Hello, \"World\"!", - "path": "C:\\Users\\test", - "newlines": "line1\nline2\nline3", - "unicode": "こんにちは 🌍", - "empty": "" - }))), - ); - - test_parity("data.message", ctx.clone(), None, None).await?; - test_parity("data.path", ctx.clone(), None, None).await?; - test_parity("data.newlines.split('\\n').length", ctx.clone(), None, None).await?; - test_parity("data.unicode", ctx.clone(), None, None).await?; - test_parity("data.empty.length", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_boolean_coercion_edge_cases() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - ctx.insert( - "emptyString".to_string(), - Arc::new(to_raw_value(&json!(""))), - ); - ctx.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); - ctx.insert( - "falseVal".to_string(), - Arc::new(to_raw_value(&json!(false))), - ); - ctx.insert("emptyArr".to_string(), Arc::new(to_raw_value(&json!([])))); - ctx.insert("emptyObj".to_string(), Arc::new(to_raw_value(&json!({})))); - - // Truthy/falsy checks - test_parity("!!zero", ctx.clone(), None, None).await?; - test_parity("!!emptyString", ctx.clone(), None, None).await?; - test_parity("!!nullVal", ctx.clone(), None, None).await?; - test_parity("!!falseVal", ctx.clone(), None, None).await?; - test_parity("!!emptyArr", ctx.clone(), None, None).await?; // [] is truthy! - test_parity("!!emptyObj", ctx.clone(), None, None).await?; // {} is truthy! - - // Logical operators with falsy values - test_parity("zero || 'default'", ctx.clone(), None, None).await?; - test_parity("zero ?? 'default'", ctx.clone(), None, None).await?; // 0 is not nullish - test_parity("nullVal ?? 'default'", ctx.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // PREVIOUS_RESULT SPECIAL HANDLING - // ========================================================================= - - #[tokio::test] - async fn parity_previous_result_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("previous_result", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "previous_result.length", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "previous_result[4].key", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // REAL-WORLD FLOW SCENARIOS - // ========================================================================= - - #[tokio::test] - async fn parity_scenario_api_pagination() -> anyhow::Result<()> { - // Simulate a flow that fetches paginated data - let mut ctx = HashMap::new(); - ctx.insert( - "fetch_result".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [{"id": 1}, {"id": 2}, {"id": 3}], - "nextCursor": "abc123", - "hasMore": true - }))), - ); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [{"id": 1}, {"id": 2}, {"id": 3}], - "nextCursor": "abc123", - "hasMore": true - }))), - ); - - // Iterator for next page - test_parity( - "previous_result.hasMore ? [previous_result.nextCursor] : []", - ctx.clone(), - None, - None, - ) - .await?; - - // Accumulating results - test_parity("fetch_result.items", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_data_transformation_pipeline() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - - // Step 1: Raw data - ctx.insert( - "raw_data".to_string(), - Arc::new(to_raw_value(&json!({ - "records": [ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-16", "amount": 50, "type": "debit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"}, - {"date": "2024-01-18", "amount": 75, "type": "debit"} - ] - }))), - ); - - // Step 2: Filter credits - ctx.insert( - "credits".to_string(), - Arc::new(to_raw_value(&json!([ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"} - ]))), - ); - - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!([ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"} - ]))), - ); - - // Filter expression - test_parity( - "raw_data.records.filter(r => r.type === 'credit')", - ctx.clone(), - None, - None, - ) - .await?; - - // Sum expression - test_parity( - "credits.reduce((sum, r) => sum + r.amount, 0)", - ctx.clone(), - None, - None, - ) - .await?; - - // Summary - test_parity( - r#" - ({ - totalCredits: credits.reduce((sum, r) => sum + r.amount, 0), - count: credits.length, - average: credits.reduce((sum, r) => sum + r.amount, 0) / credits.length - }) - "#, - ctx.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_conditional_workflow() -> anyhow::Result<()> { - // Simulate a workflow with conditional logic - let mut ctx = HashMap::new(); - ctx.insert( - "check_result".to_string(), - Arc::new(to_raw_value(&json!({ - "passed": true, - "score": 95 - }))), - ); - ctx.insert( - "user_data".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test_user", - "level": "admin" - }))), - ); - - // Branch condition - test_parity( - "check_result.passed && check_result.score > 90", - ctx.clone(), - None, - None, - ) - .await?; - - // Skip condition - test_parity( - "!check_result.passed || check_result.score < 50", - ctx.clone(), - None, - None, - ) - .await?; - - // Decision logic - test_parity( - "check_result.passed && user_data.level === 'admin' ? 'approved' : 'pending'", - ctx.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // COMPREHENSIVE OPTIONAL CHAINING TESTS - // ========================================================================= - - #[tokio::test] - async fn parity_optional_chaining_method_calls() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "data": { - "items": [1, 2, 3], - "name": "test" - } - }))), - ); - env.insert("nullObj".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert( - "undefinedField".to_string(), - Arc::new(to_raw_value(&serde_json::Value::Null)), - ); - - // Optional chaining on method calls - test_parity("obj?.data?.items?.map(x => x * 2)", env.clone(), None, None).await?; - test_parity( - "obj?.data?.items?.filter(x => x > 1)", - env.clone(), - None, - None, - ) - .await?; - test_parity("obj?.data?.items?.join(',')", env.clone(), None, None).await?; - test_parity("obj?.data?.name?.toUpperCase()", env.clone(), None, None).await?; - test_parity("obj?.data?.name?.split('')", env.clone(), None, None).await?; - - // Optional method calls on null/undefined - test_parity("nullObj?.items?.map(x => x)", env.clone(), None, None).await?; - test_parity("obj?.missing?.method?.()", env.clone(), None, None).await?; - test_parity("undefinedField?.toString?.()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_computed_properties() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "users": { - "user1": {"name": "Alice", "age": 30}, - "user2": {"name": "Bob", "age": 25} - }, - "items": ["a", "b", "c"] - }))), - ); - env.insert("key".to_string(), Arc::new(to_raw_value(&json!("user1")))); - env.insert("index".to_string(), Arc::new(to_raw_value(&json!(1)))); - env.insert("nullData".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining with computed property access - test_parity("data?.users?.[key]", env.clone(), None, None).await?; - test_parity("data?.users?.[key]?.name", env.clone(), None, None).await?; - test_parity("data?.items?.[index]", env.clone(), None, None).await?; - test_parity("data?.users?.['user2']?.age", env.clone(), None, None).await?; - - // Computed access with null/undefined - test_parity("nullData?.users?.[key]", env.clone(), None, None).await?; - test_parity("data?.missing?.[key]", env.clone(), None, None).await?; - test_parity( - "data?.users?.['nonexistent']?.name", - env.clone(), - None, - None, - ) - .await?; - - // Dynamic key access - test_parity( - "data?.users?.[`user${index + 1}`]?.name", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_function_calls() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "config".to_string(), - Arc::new(to_raw_value(&json!({ - "callback": null, - "formatter": null, - "value": 42 - }))), - ); - env.insert( - "nullConfig".to_string(), - Arc::new(to_raw_value(&json!(null))), - ); - - // Optional function call syntax - test_parity("config?.callback?.()", env.clone(), None, None).await?; - test_parity("config?.formatter?.('test')", env.clone(), None, None).await?; - test_parity("nullConfig?.callback?.()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_deep_nesting() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "response".to_string(), - Arc::new(to_raw_value(&json!({ - "data": { - "result": { - "items": [ - { - "details": { - "metadata": { - "tags": ["tag1", "tag2"] - } - } - } - ] - } - } - }))), - ); - env.insert( - "emptyResponse".to_string(), - Arc::new(to_raw_value(&json!({}))), - ); - - // Deep optional chaining - test_parity( - "response?.data?.result?.items?.[0]?.details?.metadata?.tags", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "response?.data?.result?.items?.[0]?.details?.metadata?.tags?.[0]", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "response?.data?.result?.items?.[1]?.details?.metadata?.tags", - env.clone(), - None, - None, - ) - .await?; - - // Deep chaining with missing intermediate - test_parity( - "emptyResponse?.data?.result?.items?.[0]", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "response?.data?.missing?.items?.[0]?.details", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_with_operators() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "user".to_string(), - Arc::new(to_raw_value(&json!({ - "profile": { - "settings": { - "theme": "dark", - "notifications": true - } - }, - "scores": [85, 90, 78] - }))), - ); - env.insert("nullUser".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining with nullish coalescing - test_parity( - "user?.profile?.settings?.theme ?? 'light'", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "user?.profile?.settings?.language ?? 'en'", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "nullUser?.profile?.theme ?? 'default'", - env.clone(), - None, - None, - ) - .await?; - - // Optional chaining with logical OR - test_parity( - "user?.profile?.settings?.disabled || false", - env.clone(), - None, - None, - ) - .await?; - test_parity("user?.name || 'Anonymous'", env.clone(), None, None).await?; - - // Optional chaining with logical AND - test_parity( - "user?.profile?.settings?.notifications && 'enabled'", - env.clone(), - None, - None, - ) - .await?; - - // Optional chaining in ternary - test_parity( - "user?.profile?.settings?.theme === 'dark' ? 'Dark Mode' : 'Light Mode'", - env.clone(), - None, - None, - ) - .await?; - test_parity("nullUser?.active ? 'yes' : 'no'", env.clone(), None, None).await?; - - // Optional chaining with arithmetic - test_parity("(user?.scores?.[0] ?? 0) + 10", env.clone(), None, None).await?; - test_parity("user?.scores?.length ?? 0", env.clone(), None, None).await?; - - // Optional chaining with comparison - test_parity("user?.scores?.[0] > 80", env.clone(), None, None).await?; - test_parity("nullUser?.scores?.[0] > 80", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_with_array_methods() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "users": [ - {"id": 1, "name": "Alice", "active": true}, - {"id": 2, "name": "Bob", "active": false}, - {"id": 3, "name": "Charlie", "active": true} - ] - }))), - ); - env.insert("emptyData".to_string(), Arc::new(to_raw_value(&json!({})))); - - // Optional chaining before array methods - test_parity( - "data?.users?.filter(u => u.active)", - env.clone(), - None, - None, - ) - .await?; - test_parity("data?.users?.map(u => u.name)", env.clone(), None, None).await?; - test_parity( - "data?.users?.find(u => u.id === 2)?.name", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "data?.users?.findIndex(u => u.id === 2)", - env.clone(), - None, - None, - ) - .await?; - test_parity("data?.users?.some(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.users?.every(u => u.active)", env.clone(), None, None).await?; - test_parity( - "data?.users?.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", - env.clone(), - None, - None, - ) - .await?; - - // Optional chaining on missing arrays - test_parity( - "emptyData?.users?.filter(u => u.active)", - env.clone(), - None, - None, - ) - .await?; - test_parity("data?.items?.map(i => i.value)", env.clone(), None, None).await?; - - // Chained optional access on array results - test_parity( - "data?.users?.filter(u => u.active)?.[0]?.name", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "data?.users?.filter(u => u.id > 10)?.[0]?.name ?? 'Not found'", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_in_template_literals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "person".to_string(), - Arc::new(to_raw_value(&json!({ - "firstName": "John", - "lastName": "Doe", - "address": { - "city": "NYC", - "country": "USA" - } - }))), - ); - env.insert( - "nullPerson".to_string(), - Arc::new(to_raw_value(&json!(null))), - ); - - // Template literals with optional chaining - test_parity( - "`Hello, ${person?.firstName ?? 'Guest'}!`", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "`${person?.firstName} ${person?.lastName}`", - env.clone(), - None, - None, - ) - .await?; - test_parity("`Location: ${person?.address?.city ?? 'Unknown'}, ${person?.address?.country ?? 'Unknown'}`", env.clone(), None, None).await?; - test_parity( - "`User: ${nullPerson?.name ?? 'Anonymous'}`", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_flow_context() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Optional chaining on step results - test_parity("a?.toString()", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "b?.data?.users?.[0]?.name", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "b?.data?.users?.find(u => u.id === 999)?.name", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "b?.data?.users?.find(u => u.id === 999)?.name ?? 'Not found'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Optional chaining on missing nested properties (step 'b' exists but nested path may not) - test_parity( - "b?.missing?.nested?.value ?? 'default'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Optional chaining on flow_input - test_parity( - "flow_input?.limit ?? 100", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "flow_input?.missing?.nested?.value ?? 'fallback'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Optional chaining on previous_result - test_parity( - "previous_result?.items?.[0]", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "previous_result?.missing ?? []", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_edge_cases() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - env.insert("emptyStr".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert( - "falseVal".to_string(), - Arc::new(to_raw_value(&json!(false))), - ); - env.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!({ - "zero": 0, - "empty": "", - "false": false, - "null": null, - "obj": {} - }))), - ); - - // Optional chaining preserves falsy values (except null/undefined) - test_parity("zero?.toString()", env.clone(), None, None).await?; - test_parity("emptyStr?.length", env.clone(), None, None).await?; - test_parity("falseVal?.toString()", env.clone(), None, None).await?; - test_parity("nullVal?.toString()", env.clone(), None, None).await?; - - // Difference between ?. and && - test_parity("nested?.zero", env.clone(), None, None).await?; - test_parity("nested?.empty", env.clone(), None, None).await?; - test_parity("nested?.false", env.clone(), None, None).await?; - test_parity("nested?.null", env.clone(), None, None).await?; - test_parity("nested?.null?.value", env.clone(), None, None).await?; - - // Empty object access - test_parity("nested?.obj?.missing", env.clone(), None, None).await?; - test_parity( - "nested?.obj?.missing ?? 'not there'", - env.clone(), - None, - None, - ) - .await?; - - // Chaining after primitives (should return undefined) - test_parity("nested?.zero?.value", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_typeof() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "a": {"b": {"c": 1}} - }))), - ); - env.insert("nullObj".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining in expressions that return values - test_parity("obj?.a?.b?.c", env.clone(), None, None).await?; - test_parity("obj?.x?.y?.z", env.clone(), None, None).await?; - test_parity("nullObj?.a?.b?.c", env.clone(), None, None).await?; - - // Check that optional chaining works with typeof - test_parity("typeof obj?.a?.b?.c", env.clone(), None, None).await?; - test_parity("typeof obj?.missing?.value", env.clone(), None, None).await?; - test_parity("typeof nullObj?.value", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASE TESTS: Potential breaking changes between engines - // ========================================================================= - - #[tokio::test] - async fn parity_large_integers() -> anyhow::Result<()> { - let mut env = HashMap::new(); - - // i32 boundary values - env.insert( - "i32_max".to_string(), - Arc::new(to_raw_value(&json!(2147483647))), // i32::MAX - ); - env.insert( - "i32_max_plus_1".to_string(), - Arc::new(to_raw_value(&json!(2147483648_i64))), // i32::MAX + 1 - ); - env.insert( - "i32_min".to_string(), - Arc::new(to_raw_value(&json!(-2147483648))), // i32::MIN - ); - env.insert( - "i32_min_minus_1".to_string(), - Arc::new(to_raw_value(&json!(-2147483649_i64))), // i32::MIN - 1 - ); - - // Typical timestamp (milliseconds since epoch) - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // Jan 1, 2024 - ); - - // Near MAX_SAFE_INTEGER - env.insert( - "large_safe".to_string(), - Arc::new(to_raw_value(&json!(9007199254740991_i64))), // MAX_SAFE_INTEGER - ); - - // Basic operations with i32 boundary values - test_parity("i32_max", env.clone(), None, None).await?; - test_parity("i32_max + 1", env.clone(), None, None).await?; - test_parity("i32_max_plus_1", env.clone(), None, None).await?; - test_parity("i32_max_plus_1 + 1", env.clone(), None, None).await?; - test_parity("i32_min", env.clone(), None, None).await?; - test_parity("i32_min - 1", env.clone(), None, None).await?; - test_parity("i32_min_minus_1", env.clone(), None, None).await?; - - // Timestamp arithmetic - test_parity("timestamp", env.clone(), None, None).await?; - test_parity("timestamp + 86400000", env.clone(), None, None).await?; // +1 day - test_parity("timestamp - 3600000", env.clone(), None, None).await?; // -1 hour - - // MAX_SAFE_INTEGER operations - test_parity("large_safe", env.clone(), None, None).await?; - test_parity("large_safe - 1", env.clone(), None, None).await?; - - // Comparisons at boundaries - test_parity("i32_max === 2147483647", env.clone(), None, None).await?; - test_parity("i32_max_plus_1 === 2147483648", env.clone(), None, None).await?; - test_parity("timestamp > 1704067200000", env.clone(), None, None).await?; - test_parity("timestamp === 1704067200000", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_sparse_arrays() -> anyhow::Result<()> { - let mut env = HashMap::new(); - - // Sparse arrays are tricky - we'll simulate them via expressions - // Note: JSON doesn't support sparse arrays directly, so we test via JS - - // Regular array for comparison - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Test array operations that might behave differently with holes - test_parity("[1, 2, 3].length", env.clone(), None, None).await?; - test_parity("[1, 2, 3].map(x => x * 2)", env.clone(), None, None).await?; - test_parity("[1, 2, 3].filter(x => x > 1)", env.clone(), None, None).await?; - test_parity( - "[1, 2, 3].reduce((a, b) => a + b, 0)", - env.clone(), - None, - None, - ) - .await?; - - // Array with undefined values (different from holes) - test_parity( - "[1, undefined, 3].map(x => x ?? 'missing')", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "[1, null, 3].map(x => x ?? 'missing')", - env.clone(), - None, - None, - ) - .await?; - - // Array.from behavior - test_parity("Array.from([1, 2, 3])", env.clone(), None, None).await?; - test_parity( - "Array.from({length: 3}, (_, i) => i)", - env.clone(), - None, - None, - ) - .await?; - - // Spread operator - test_parity("[...arr]", env.clone(), None, None).await?; - test_parity("[...arr, 6, 7]", env.clone(), None, None).await?; - test_parity("[0, ...arr]", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_unicode_and_emoji() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("emoji".to_string(), Arc::new(to_raw_value(&json!("🎉")))); - env.insert( - "text_with_emoji".to_string(), - Arc::new(to_raw_value(&json!("Hello 🌍 World!"))), - ); - env.insert("cafe".to_string(), Arc::new(to_raw_value(&json!("café")))); - env.insert( - "chinese".to_string(), - Arc::new(to_raw_value(&json!("你好世界"))), - ); - env.insert( - "mixed".to_string(), - Arc::new(to_raw_value(&json!("Hello 世界 🌍"))), - ); - - // String length (surrogate pairs count as 2) - test_parity("emoji.length", env.clone(), None, None).await?; - test_parity("text_with_emoji.length", env.clone(), None, None).await?; - test_parity("cafe.length", env.clone(), None, None).await?; - test_parity("chinese.length", env.clone(), None, None).await?; - - // String operations - test_parity("emoji.charCodeAt(0)", env.clone(), None, None).await?; - test_parity("text_with_emoji.indexOf('🌍')", env.clone(), None, None).await?; - test_parity("text_with_emoji.includes('🌍')", env.clone(), None, None).await?; - - // Substring operations - test_parity("text_with_emoji.substring(0, 5)", env.clone(), None, None).await?; - test_parity("text_with_emoji.slice(-1)", env.clone(), None, None).await?; - - // String comparison - test_parity("cafe === 'café'", env.clone(), None, None).await?; - test_parity("'café' === 'café'", env.clone(), None, None).await?; - - // Template literals with unicode - test_parity("`Hello ${emoji}`", env.clone(), None, None).await?; - test_parity("`${chinese} - ${emoji}`", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_numeric_values() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - env.insert( - "negative_zero_str".to_string(), - Arc::new(to_raw_value(&json!("-0"))), - ); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - - // Basic numeric operations - test_parity("0 === 0", env.clone(), None, None).await?; - test_parity("-0 === 0", env.clone(), None, None).await?; - test_parity("zero === 0", env.clone(), None, None).await?; - - // Division by zero - test_parity("1 / 0", env.clone(), None, None).await?; // Infinity -> null in JSON - test_parity("-1 / 0", env.clone(), None, None).await?; // -Infinity -> null in JSON - test_parity("0 / 0", env.clone(), None, None).await?; // NaN -> null in JSON - - // NaN checks - test_parity("Number.isNaN(0 / 0)", env.clone(), None, None).await?; - test_parity("Number.isFinite(1 / 0)", env.clone(), None, None).await?; - test_parity("Number.isFinite(num)", env.clone(), None, None).await?; - - // Safe integer checks - test_parity("Number.isSafeInteger(42)", env.clone(), None, None).await?; - test_parity( - "Number.isSafeInteger(9007199254740991)", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "Number.isSafeInteger(9007199254740992)", - env.clone(), - None, - None, - ) - .await?; - - // Number parsing - test_parity("parseInt('42')", env.clone(), None, None).await?; - test_parity("parseFloat('3.14')", env.clone(), None, None).await?; - test_parity("parseInt('0xff', 16)", env.clone(), None, None).await?; - test_parity("parseInt('101', 2)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_property_order() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "z": 3, - "a": 1, - "m": 2, - "b": 4 - }))), - ); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!({ - "outer": { - "z": 1, - "a": 2 - } - }))), - ); - - // Object.keys, Object.values, Object.entries - // Note: Order might differ but we compare as sets - test_parity("Object.keys(obj).sort()", env.clone(), None, None).await?; - test_parity( - "Object.values(obj).sort((a, b) => a - b)", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))", - env.clone(), - None, - None, - ) - .await?; - - // Object spread (order might differ) - test_parity("{...obj, extra: 5}", env.clone(), None, None).await?; - test_parity("{first: 0, ...obj}", env.clone(), None, None).await?; - - // Nested object access - test_parity("nested.outer.z", env.clone(), None, None).await?; - test_parity("Object.keys(nested.outer).sort()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_prototype_methods() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello world"))), - ); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - // Array methods - test_parity("arr.includes(3)", env.clone(), None, None).await?; - test_parity("arr.indexOf(3)", env.clone(), None, None).await?; - test_parity("arr.lastIndexOf(3)", env.clone(), None, None).await?; - test_parity("arr.find(x => x > 3)", env.clone(), None, None).await?; - test_parity("arr.findIndex(x => x > 3)", env.clone(), None, None).await?; - test_parity("arr.every(x => x > 0)", env.clone(), None, None).await?; - test_parity("arr.some(x => x > 4)", env.clone(), None, None).await?; - test_parity("arr.flat()", env.clone(), None, None).await?; - test_parity("arr.flatMap(x => [x, x * 2])", env.clone(), None, None).await?; - test_parity("arr.fill(0, 1, 3)", env.clone(), None, None).await?; - test_parity("[...arr].reverse()", env.clone(), None, None).await?; - - // String methods - test_parity("str.split(' ')", env.clone(), None, None).await?; - test_parity("str.toUpperCase()", env.clone(), None, None).await?; - test_parity("str.toLowerCase()", env.clone(), None, None).await?; - test_parity("str.trim()", env.clone(), None, None).await?; - test_parity("str.padStart(15, '_')", env.clone(), None, None).await?; - test_parity("str.padEnd(15, '_')", env.clone(), None, None).await?; - test_parity("str.startsWith('hello')", env.clone(), None, None).await?; - test_parity("str.endsWith('world')", env.clone(), None, None).await?; - test_parity("str.repeat(2)", env.clone(), None, None).await?; - - // Object methods - test_parity("Object.keys(obj)", env.clone(), None, None).await?; - test_parity("Object.values(obj)", env.clone(), None, None).await?; - test_parity("Object.entries(obj)", env.clone(), None, None).await?; - test_parity("Object.assign({}, obj, {c: 3})", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_basic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("Hello123World456"))), - ); - env.insert( - "email".to_string(), - Arc::new(to_raw_value(&json!("test@example.com"))), - ); - - // Basic regex matching - test_parity("/\\d+/.test(text)", env.clone(), None, None).await?; - test_parity("text.match(/\\d+/)", env.clone(), None, None).await?; - test_parity("text.match(/\\d+/g)", env.clone(), None, None).await?; - - // Replace with regex - test_parity("text.replace(/\\d+/, 'X')", env.clone(), None, None).await?; - test_parity("text.replace(/\\d+/g, 'X')", env.clone(), None, None).await?; - - // Split with regex - test_parity("text.split(/\\d+/)", env.clone(), None, None).await?; - - // Case insensitive - test_parity("/hello/i.test(text)", env.clone(), None, None).await?; - test_parity("text.match(/hello/i)", env.clone(), None, None).await?; - - // Email validation (basic pattern) - test_parity( - "/^[^@]+@[^@]+\\.[^@]+$/.test(email)", - env.clone(), - None, - None, - ) - .await?; - - // Capturing groups (basic) - test_parity("text.match(/(\\d+)/)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // 2024-01-01T00:00:00Z - ); - env.insert( - "iso_date".to_string(), - Arc::new(to_raw_value(&json!("2024-01-15T12:30:00.000Z"))), - ); - - // Date parsing - test_parity("Date.parse(iso_date)", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getTime()", env.clone(), None, None).await?; - test_parity("new Date(timestamp).toISOString()", env.clone(), None, None).await?; - - // UTC methods (timezone-independent) - test_parity( - "new Date(iso_date).getUTCFullYear()", - env.clone(), - None, - None, - ) - .await?; - test_parity("new Date(iso_date).getUTCMonth()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCDate()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCHours()", env.clone(), None, None).await?; - test_parity( - "new Date(iso_date).getUTCMinutes()", - env.clone(), - None, - None, - ) - .await?; - - // Date arithmetic - test_parity( - "new Date(timestamp + 86400000).toISOString()", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "new Date(timestamp - 3600000).toISOString()", - env.clone(), - None, - None, - ) - .await?; - - // Date comparison - test_parity("new Date(iso_date).getTime() > 0", env.clone(), None, None).await?; - test_parity( - "new Date(iso_date).getTime() === Date.parse(iso_date)", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_handling() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Try-catch with safe access (using step 'b' which has data) - test_parity( - "(() => { try { return b.data.total; } catch(e) { return 'error'; } })()", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Error from invalid JSON parse (should be caught) - test_parity( - "(() => { try { return JSON.parse('invalid'); } catch(e) { return 'parse_error'; } })()", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Typeof for error prevention - test_parity( - "typeof b.missing === 'undefined'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "typeof b.data.total === 'number'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Ternary with type checks - test_parity( - "typeof b === 'object' && b !== null ? b.data.total : 'fallback'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_set_and_map() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3]))), - ); - env.insert( - "pairs".to_string(), - Arc::new(to_raw_value(&json!([["a", 1], ["b", 2], ["c", 3]]))), - ); - - // Set operations (converted back to array for JSON) - test_parity("[...new Set(arr)]", env.clone(), None, None).await?; - test_parity("new Set(arr).size", env.clone(), None, None).await?; - test_parity("new Set(arr).has(2)", env.clone(), None, None).await?; - test_parity("new Set(arr).has(5)", env.clone(), None, None).await?; - - // Map operations (converted back for JSON) - test_parity("new Map(pairs).get('a')", env.clone(), None, None).await?; - test_parity("new Map(pairs).has('b')", env.clone(), None, None).await?; - test_parity("new Map(pairs).size", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).keys()]", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).values()]", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).entries()]", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_computed_property_names() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "key".to_string(), - Arc::new(to_raw_value(&json!("dynamicKey"))), - ); - env.insert("prefix".to_string(), Arc::new(to_raw_value(&json!("item")))); - env.insert("index".to_string(), Arc::new(to_raw_value(&json!(42)))); - - // Computed property access - test_parity("({a: 1, b: 2})[key] ?? 'missing'", env.clone(), None, None).await?; - test_parity("({dynamicKey: 'found'})[key]", env.clone(), None, None).await?; - - // Computed property creation - test_parity("({[key]: 'value'})", env.clone(), None, None).await?; - test_parity("({[prefix + '_' + index]: true})", env.clone(), None, None).await?; - test_parity( - "({[`${prefix}_${index}`]: 'computed'})", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_destructuring_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "user": {"name": "Alice", "age": 30}, - "items": [1, 2, 3, 4, 5], - "meta": {"count": 5} - }))), - ); - - // Nested destructuring - test_parity("(({user: {name}}) => name)(data)", env.clone(), None, None).await?; - test_parity( - "(({items: [first, second, ...rest]}) => ({first, second, rest}))(data)", - env.clone(), - None, - None, - ) - .await?; - - // Default values in destructuring - test_parity( - "(({missing = 'default'}) => missing)(data)", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "(({user: {nickname = 'unknown'}}) => nickname)(data)", - env.clone(), - None, - None, - ) - .await?; - - // Renaming in destructuring - test_parity("(({user: u}) => u.name)(data)", env.clone(), None, None).await?; - test_parity( - "(({meta: {count: total}}) => total)(data)", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_arrow_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "numbers".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - env.insert( - "users".to_string(), - Arc::new(to_raw_value(&json!([ - {"name": "Alice", "score": 85}, - {"name": "Bob", "score": 92}, - {"name": "Charlie", "score": 78} - ]))), - ); - - // Simple arrow functions - test_parity("numbers.map(x => x * 2)", env.clone(), None, None).await?; - test_parity("numbers.filter(x => x > 2)", env.clone(), None, None).await?; - test_parity( - "numbers.reduce((a, b) => a + b, 0)", - env.clone(), - None, - None, - ) - .await?; - - // Arrow functions with objects - test_parity("users.map(u => u.name)", env.clone(), None, None).await?; - test_parity("users.filter(u => u.score >= 80)", env.clone(), None, None).await?; - test_parity("users.find(u => u.name === 'Bob')", env.clone(), None, None).await?; - - // Arrow functions returning objects (note the parentheses) - test_parity( - "numbers.map(x => ({value: x, doubled: x * 2}))", - env.clone(), - None, - None, - ) - .await?; - - // Chained arrow function calls - test_parity( - "numbers.filter(x => x > 1).map(x => x * 10)", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "users.filter(u => u.score > 80).map(u => u.name)", - env.clone(), - None, - None, - ) - .await?; - - // Arrow function with multiple params - test_parity( - "numbers.reduce((sum, val) => sum + val, 0)", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "numbers.map((val, idx) => ({index: idx, value: val}))", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_type_coercion() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("str_num".to_string(), Arc::new(to_raw_value(&json!("42")))); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert( - "bool_true".to_string(), - Arc::new(to_raw_value(&json!(true))), - ); - env.insert( - "bool_false".to_string(), - Arc::new(to_raw_value(&json!(false))), - ); - env.insert("null_val".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("empty_str".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert("empty_arr".to_string(), Arc::new(to_raw_value(&json!([])))); - env.insert("empty_obj".to_string(), Arc::new(to_raw_value(&json!({})))); - - // String to number - test_parity("Number(str_num)", env.clone(), None, None).await?; - test_parity("+str_num", env.clone(), None, None).await?; - test_parity("parseInt(str_num)", env.clone(), None, None).await?; - - // Number to string - test_parity("String(num)", env.clone(), None, None).await?; - test_parity("num.toString()", env.clone(), None, None).await?; - test_parity("'' + num", env.clone(), None, None).await?; - - // Truthy/falsy checks - test_parity("!!str_num", env.clone(), None, None).await?; - test_parity("!!empty_str", env.clone(), None, None).await?; - test_parity("!!null_val", env.clone(), None, None).await?; - test_parity("!!empty_arr", env.clone(), None, None).await?; - test_parity("!!empty_obj", env.clone(), None, None).await?; - - // Boolean operations - test_parity("bool_true && 'yes'", env.clone(), None, None).await?; - test_parity("bool_false || 'no'", env.clone(), None, None).await?; - test_parity("null_val ?? 'default'", env.clone(), None, None).await?; - test_parity("empty_str || 'fallback'", env.clone(), None, None).await?; - test_parity("empty_str ?? 'wont_use'", env.clone(), None, None).await?; // empty string is not nullish - - // Array coercion - test_parity("Boolean(empty_arr)", env.clone(), None, None).await?; // empty array is truthy - test_parity("empty_arr.length || 'empty'", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_json_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test", - "values": [1, 2, 3], - "nested": {"key": "value"} - }))), - ); - env.insert( - "json_str".to_string(), - Arc::new(to_raw_value(&json!(r#"{"parsed": true, "count": 42}"#))), - ); - - // JSON.stringify - test_parity("JSON.stringify(obj)", env.clone(), None, None).await?; - test_parity("JSON.stringify(obj, null, 2)", env.clone(), None, None).await?; - test_parity("JSON.stringify([1, 2, 3])", env.clone(), None, None).await?; - test_parity("JSON.stringify(null)", env.clone(), None, None).await?; - test_parity("JSON.stringify('string')", env.clone(), None, None).await?; - - // JSON.parse - test_parity("JSON.parse(json_str)", env.clone(), None, None).await?; - test_parity("JSON.parse(json_str).parsed", env.clone(), None, None).await?; - test_parity("JSON.parse(json_str).count", env.clone(), None, None).await?; - - // Round-trip - test_parity( - "JSON.parse(JSON.stringify(obj)).name", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "JSON.parse(JSON.stringify(obj)).values", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_math_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(16)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(-5.7)))); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9]))), - ); - - // Basic Math functions - test_parity("Math.abs(y)", env.clone(), None, None).await?; - test_parity("Math.sqrt(x)", env.clone(), None, None).await?; - test_parity("Math.pow(2, 10)", env.clone(), None, None).await?; - test_parity("Math.floor(y)", env.clone(), None, None).await?; - test_parity("Math.ceil(y)", env.clone(), None, None).await?; - test_parity("Math.round(y)", env.clone(), None, None).await?; - test_parity("Math.trunc(y)", env.clone(), None, None).await?; - - // Min/Max - test_parity("Math.min(3, 1, 4)", env.clone(), None, None).await?; - test_parity("Math.max(3, 1, 4)", env.clone(), None, None).await?; - test_parity("Math.min(...arr)", env.clone(), None, None).await?; - test_parity("Math.max(...arr)", env.clone(), None, None).await?; - - // Trigonometric (with rounding to avoid precision issues) - test_parity( - "Math.round(Math.sin(0) * 1000) / 1000", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "Math.round(Math.cos(0) * 1000) / 1000", - env.clone(), - None, - None, - ) - .await?; - - // Logarithmic - test_parity("Math.log(1)", env.clone(), None, None).await?; - test_parity("Math.log10(100)", env.clone(), None, None).await?; - test_parity("Math.log2(8)", env.clone(), None, None).await?; - - // Constants - test_parity("Math.round(Math.PI * 1000) / 1000", env.clone(), None, None).await?; - test_parity("Math.round(Math.E * 1000) / 1000", env.clone(), None, None).await?; - - // Sign and other - test_parity("Math.sign(-5)", env.clone(), None, None).await?; - test_parity("Math.sign(5)", env.clone(), None, None).await?; - test_parity("Math.sign(0)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_bitwise_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("a".to_string(), Arc::new(to_raw_value(&json!(0b1010)))); - env.insert("b".to_string(), Arc::new(to_raw_value(&json!(0b1100)))); - env.insert("neg".to_string(), Arc::new(to_raw_value(&json!(-1)))); - - // Basic bitwise operations - test_parity("a & b", env.clone(), None, None).await?; - test_parity("a | b", env.clone(), None, None).await?; - test_parity("a ^ b", env.clone(), None, None).await?; - test_parity("~a", env.clone(), None, None).await?; - - // Shifts - test_parity("a << 2", env.clone(), None, None).await?; - test_parity("a >> 1", env.clone(), None, None).await?; - test_parity("neg >>> 0", env.clone(), None, None).await?; // unsigned right shift - - // Combined operations - test_parity("(a & b) | 1", env.clone(), None, None).await?; - test_parity("a ^ b ^ a", env.clone(), None, None).await?; // should equal b - - Ok(()) - } - - #[tokio::test] - async fn parity_array_slice_splice() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Slice (non-mutating) - test_parity("arr.slice()", env.clone(), None, None).await?; - test_parity("arr.slice(1)", env.clone(), None, None).await?; - test_parity("arr.slice(1, 3)", env.clone(), None, None).await?; - test_parity("arr.slice(-2)", env.clone(), None, None).await?; - test_parity("arr.slice(-3, -1)", env.clone(), None, None).await?; - test_parity("arr.slice(1, -1)", env.clone(), None, None).await?; - - // Concat (non-mutating) - test_parity("arr.concat([6, 7])", env.clone(), None, None).await?; - test_parity("arr.concat([6], [7, 8])", env.clone(), None, None).await?; - test_parity("[].concat(arr, [6])", env.clone(), None, None).await?; - - // Join - test_parity("arr.join()", env.clone(), None, None).await?; - test_parity("arr.join('-')", env.clone(), None, None).await?; - test_parity("arr.join('')", env.clone(), None, None).await?; - - // Copy and splice (to avoid mutating original) - test_parity("[...arr].splice(1, 2)", env.clone(), None, None).await?; - test_parity( - "(() => { const a = [...arr]; a.splice(1, 2, 'x'); return a; })()", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_error_extraction() -> anyhow::Result<()> { - // Test the specific error extraction logic used in flows - // Use existing context and add parallel results step 'h' - let (mut ctx, fi, fe) = create_multi_step_flow_context(); - - // Simulated parallel results with one error - add as step 'h' - ctx.insert( - "h".to_string(), - Arc::new(to_raw_value(&json!([ - {"success": true, "data": "result1"}, - {"error": {"message": "Something failed", "code": 500}}, - {"success": true, "data": "result3"} - ]))), - ); - - // Find error in step h's results - test_parity( - "h.find(r => r.error)?.error", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity( - "h.filter(r => r.error).length", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - test_parity("h.some(r => r.error)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity( - "h.every(r => !r.error)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Extract all successful results - test_parity( - "h.filter(r => r.success).map(r => r.data)", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_string_template_complex() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "user".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "Alice", - "email": "alice@example.com", - "score": 95.5 - }))), - ); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!(["apple", "banana", "cherry"]))), - ); - - // Nested expressions in templates - test_parity( - "`User: ${user.name} (${user.email})`", - env.clone(), - None, - None, - ) - .await?; - test_parity("`Score: ${user.score.toFixed(1)}`", env.clone(), None, None).await?; - test_parity("`Items: ${items.join(', ')}`", env.clone(), None, None).await?; - test_parity("`Count: ${items.length}`", env.clone(), None, None).await?; - - // Conditional in template - test_parity( - "`Status: ${user.score >= 90 ? 'A' : 'B'}`", - env.clone(), - None, - None, - ) - .await?; - - // Method calls in template - test_parity( - "`Upper: ${user.name.toUpperCase()}`", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "`First item: ${items[0].charAt(0).toUpperCase() + items[0].slice(1)}`", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // ES2022+ METHOD AVAILABILITY TESTS - // These test methods that may not be available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_es2022_array_at() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.at() - ES2022 - test_parity("arr.at(0)", env.clone(), None, None).await?; - test_parity("arr.at(-1)", env.clone(), None, None).await?; - test_parity("arr.at(-2)", env.clone(), None, None).await?; - test_parity("arr.at(10)", env.clone(), None, None).await?; // out of bounds - - Ok(()) - } - - #[tokio::test] - async fn parity_es2022_string_at() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); - - // String.prototype.at() - ES2022 - test_parity("str.at(0)", env.clone(), None, None).await?; - test_parity("str.at(-1)", env.clone(), None, None).await?; - test_parity("str.at(-2)", env.clone(), None, None).await?; - test_parity("str.at(10)", env.clone(), None, None).await?; // out of bounds - - Ok(()) - } - - #[tokio::test] - async fn parity_es2022_object_hasown() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - // Object.hasOwn() - ES2022 - test_parity("Object.hasOwn(obj, 'a')", env.clone(), None, None).await?; - test_parity("Object.hasOwn(obj, 'c')", env.clone(), None, None).await?; - test_parity("Object.hasOwn(obj, 'toString')", env.clone(), None, None).await?; // inherited - - Ok(()) - } - - #[tokio::test] - async fn parity_es2021_string_replaceall() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("foo bar foo baz foo"))), - ); - - // String.prototype.replaceAll() - ES2021 - test_parity("str.replaceAll('foo', 'qux')", env.clone(), None, None).await?; - test_parity("str.replaceAll('x', 'y')", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_findlast() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.findLast() - ES2023 - test_parity("arr.findLast(x => x > 2)", env.clone(), None, None).await?; - test_parity("arr.findLast(x => x > 10)", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_findlastindex() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.findLastIndex() - ES2023 - test_parity("arr.findLastIndex(x => x > 2)", env.clone(), None, None).await?; - test_parity("arr.findLastIndex(x => x > 10)", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_tosorted() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5]))), - ); - - // Array.prototype.toSorted() - ES2023 (non-mutating sort) - test_parity("arr.toSorted()", env.clone(), None, None).await?; - test_parity("arr.toSorted((a, b) => b - a)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_toreversed() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.toReversed() - ES2023 (non-mutating reverse) - test_parity("arr.toReversed()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_tospliced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.toSpliced() - ES2023 (non-mutating splice) - test_parity("arr.toSpliced(1, 2)", env.clone(), None, None).await?; - test_parity("arr.toSpliced(1, 2, 'a', 'b')", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_with() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.with() - ES2023 (non-mutating index assignment) - test_parity("arr.with(2, 99)", env.clone(), None, None).await?; - test_parity("arr.with(-1, 99)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2024_object_groupby() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!([ - {"type": "fruit", "name": "apple"}, - {"type": "vegetable", "name": "carrot"}, - {"type": "fruit", "name": "banana"} - ]))), - ); - - // Object.groupBy() - ES2024 - test_parity( - "Object.groupBy(items, item => item.type)", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // REGEX FEATURE TESTS - Lookbehind and Named Groups - // These may not be available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_regex_lookbehind() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("price: $100, discount: $20"))), - ); - - // Lookbehind assertion - may not work in QuickJS - // This matches numbers that come after a $ - test_parity("text.match(/(?<=\\$)\\d+/g)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_negative_lookbehind() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("foo123 bar456"))), - ); - - // Negative lookbehind - may not work in QuickJS - test_parity("text.match(/(? anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "date".to_string(), - Arc::new(to_raw_value(&json!("2024-01-15"))), - ); - - // Named capture groups - may not work in QuickJS - test_parity( - "/(?\\d{4})-(?\\d{2})-(?\\d{2})/.exec(date)?.groups?.year", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_d_flag() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("hello world"))), - ); - - // d flag (indices) - may not work in QuickJS - test_parity("/world/d.exec(text)?.indices?.[0]", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // BROWSER/RUNTIME API TESTS - These are likely NOT available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_atob_btoa() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Base64 encoding/decoding - browser APIs, likely NOT in QuickJS - test_parity("typeof atob", env.clone(), None, None).await?; - test_parity("typeof btoa", env.clone(), None, None).await?; - // If available, test actual usage - test_parity( - "typeof btoa === 'function' ? btoa('hello') : 'not_available'", - env.clone(), - None, - None, - ) - .await?; - test_parity( - "typeof atob === 'function' ? atob('aGVsbG8=') : 'not_available'", - env.clone(), - None, - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_text_encoder_decoder() -> anyhow::Result<()> { - let env = HashMap::new(); - - // TextEncoder/TextDecoder - browser/Node APIs - test_parity("typeof TextEncoder", env.clone(), None, None).await?; - test_parity("typeof TextDecoder", env.clone(), None, None).await?; - - Ok(()) - } - - // NOTE: This test is EXPECTED to fail - Intl is NOT available in QuickJS - // Deno Core: typeof Intl = "object" - // QuickJS: typeof Intl = "undefined" - // - // BREAKING CHANGE: Any expression using Intl.NumberFormat, Intl.DateTimeFormat, - // or other Intl APIs will fail in QuickJS. - // - // #[tokio::test] - // async fn parity_intl_apis() -> anyhow::Result<()> { - // // Intl APIs are NOT available in QuickJS - this test documents the breaking change - // // Deno Core: typeof Intl = "object" - // // QuickJS: typeof Intl = "undefined" - // } - - #[tokio::test] - async fn parity_url_apis() -> anyhow::Result<()> { - let env = HashMap::new(); - - // URL and URLSearchParams - browser/Node APIs - test_parity("typeof URL", env.clone(), None, None).await?; - test_parity("typeof URLSearchParams", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // NOTE: Non-existent step access via results proxy cannot be tested in unit tests - // because the results proxy is only set up during actual flow execution (requires by_id context). - // See flow_engine_parity.rs for test_flow_results_non_existent_step which tests this behavior. - // - // IMPORTANT: Both Deno Core and QuickJS throw errors when accessing non-existent steps, - // even with optional chaining (results?.nonexistent). This is because: - // 1. results is a Proxy object (not null), so ?. doesn't short-circuit - // 2. The proxy's get handler triggers a backend lookup - // 3. The backend returns "Not found" error - // ========================================================================= -} diff --git a/backend/windmill-worker/src/js_eval_quickjs.rs b/backend/windmill-worker/src/js_eval_quickjs.rs deleted file mode 100644 index 7c59e36ced..0000000000 --- a/backend/windmill-worker/src/js_eval_quickjs.rs +++ /dev/null @@ -1,957 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -//! QuickJS-based JavaScript expression evaluation for flow transformations. -//! -//! This module provides an alternative to deno_core for evaluating arbitrary JavaScript -//! expressions in flow transformations. QuickJS offers significantly faster startup times -//! (~200μs vs ~3ms for V8), making it ideal for evaluating many small expressions. -//! -//! ## Performance Characteristics (release mode benchmarks) -//! - **Simple expressions**: ~238μs (QuickJS) vs ~3.05ms (deno_core) = **~13x faster** -//! - **Complex expressions**: ~192μs (QuickJS) vs ~3.09ms (deno_core) = **~16x faster** -//! - **Memory**: ~2.5% of V8's footprint -//! -//! For flow expression evaluation where startup time dominates, QuickJS is significantly -//! faster overall despite being slower for long-running CPU-intensive code. -//! -//! ## Async Operations -//! This implementation uses true async Rust callbacks (similar to deno_core's ops) for -//! `variable()`, `resource()`, and `results.xxx` access. The async functions use -//! rquickjs's `Async>` wrapper which returns JavaScript Promises that are -//! resolved when the Rust async operations complete. No pre-fetching is required. - -use std::collections::HashMap; -use std::sync::Arc; - -use rquickjs::{ - async_with, - prelude::{Async, Func, MutFn}, - AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object, Value, -}; -use serde_json::value::RawValue; - -use windmill_common::client::AuthedClient; -use windmill_common::flow_status::JobResult; - -use crate::js_eval::{replace_with_await, replace_with_await_result, IdContext}; - -/// Shared state for async operations within QuickJS -#[derive(Clone)] -struct AsyncOpState { - client: AuthedClient, -} - -/// Evaluates a JavaScript expression using QuickJS runtime. -/// -/// This function provides the same interface as `eval_timeout` but uses QuickJS -/// instead of deno_core/V8 for potentially faster startup times. -/// -/// Unlike deno_core, this uses true async Rust callbacks for `variable()`, -/// `resource()`, and `results.xxx` access - no pre-fetching required. -pub async fn eval_timeout_quickjs( - expr: String, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option<&HashMap>>, - authed_client: Option<&AuthedClient>, - by_id: Option<&IdContext>, - ctx: Option>, -) -> anyhow::Result> { - let expr = expr.trim().to_string(); - - tracing::debug!( - "evaluating js eval (quickjs): {} with context {:?}", - expr, - transform_context - ); - - // Clone data for the blocking task - let by_id_clone = by_id.cloned(); - let flow_input_clone = flow_input.clone(); - let flow_env_clone = flow_env.cloned(); - let authed_client_clone = authed_client.cloned(); - - // Determine which context keys are actually used in the expression - let p_ids = by_id.map(|x| { - [ - format!("results.{}", x.previous_id), - format!("results?.{}", x.previous_id), - format!("results[\"{}\"]", x.previous_id), - format!("results?.[\"{}\"]", x.previous_id), - ] - }); - - let mut context_keys: Vec = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .cloned() - .collect(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - context_keys.push("previous_result".to_string()); - } - - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - // Filter transform_context to only include used keys - let filtered_context: HashMap>> = transform_context - .into_iter() - .filter(|(k, _)| context_keys.contains(k)) - .collect(); - - let expr_clone = expr.clone(); - - // Run the QuickJS evaluation with a timeout - tokio::time::timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - // Create a new tokio runtime for async operations within the blocking context - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - rt.block_on(async move { - eval_quickjs_inner( - &expr_clone, - filtered_context, - flow_input_clone, - flow_env_clone, - authed_client_clone, - by_id_clone, - ctx, - context_keys, - ) - .await - }) - }), - ) - .await - .map_err(|_| { - anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)") - })?? -} - -/// Memory limit for QuickJS runtime (32MB). -/// This is much smaller than deno_core's 128MB limit since flow expressions -/// should be lightweight transformations, not memory-intensive operations. -const QUICKJS_MEMORY_LIMIT: usize = 32 * 1024 * 1024; - -async fn eval_quickjs_inner( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - authed_client: Option, - by_id: Option, - extra_ctx: Option>, - context_keys: Vec, -) -> anyhow::Result> { - let runtime = AsyncRuntime::new()?; - runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; - let context = AsyncContext::full(&runtime).await?; - - // Create shared state for async ops if we have a client - let op_state = authed_client.map(|client| Arc::new(AsyncOpState { client })); - - let op_state_clone = op_state.clone(); - let by_id_clone = by_id.clone(); - - // Transform expression to add await for variable/resource/results access - let expr_with_funcs = ["variable", "resource"] - .into_iter() - .fold(expr.to_string(), replace_with_await); - let transformed_expr = replace_with_await_result(expr_with_funcs); - - async_with!(context => |ctx| { - let globals = ctx.globals(); - - // Set up context variables - for key in &context_keys { - if key == "flow_input" { - if let Some(ref fi) = flow_input { - let json_str = serde_json::to_string(fi.as_ref())?; - let val: serde_json::Value = serde_json::from_str(&json_str)?; - let js_val = json_to_js(&ctx, &val)?; - globals.set(key.as_str(), js_val)?; - } else { - globals.set(key.as_str(), Value::new_null(ctx.clone()))?; - } - } else if let Some(raw_val) = transform_context.get(key) { - let val: serde_json::Value = serde_json::from_str(raw_val.get())?; - let js_val = json_to_js(&ctx, &val)?; - globals.set(key.as_str(), js_val)?; - } - } - - // Set up flow_env if referenced - if expr.contains("flow_env") { - if let Some(ref fe) = flow_env { - let obj = Object::new(ctx.clone())?; - for (k, v) in fe { - let val: serde_json::Value = serde_json::from_str(v.get())?; - let js_val = json_to_js(&ctx, &val)?; - obj.set(k.as_str(), js_val)?; - } - globals.set("flow_env", obj)?; - } else { - globals.set("flow_env", Object::new(ctx.clone())?)?; - } - } - - // Set up additional context variables - if let Some(ctx_vars) = extra_ctx { - for (k, v) in ctx_vars { - globals.set(k.as_str(), v.as_str())?; - } - } - - // Set up error extraction if needed - if expr.contains("error") && context_keys.contains(&"previous_result".to_string()) { - let error_setup = r#" - let error = previous_result?.error; - if (!error) { - if (Array.isArray(previous_result)) { - const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); - if (errors.length === 1) { - error = errors[0].error; - } else if (errors.length > 1) { - error = { - name: 'MultipleErrors', - message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), - errors: previous_result - }; - } else { - error = { - name: 'MultipleErrors', - message: "Could not parse errors", - errors: previous_result - }; - } - } else { - if (previous_result) { - error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; - } else { - error = { name: 'UnknownError', message: 'No error found' }; - } - } - } - "#; - ctx.eval::<(), _>(error_setup).catch(&ctx).map_err(quickjs_error_to_anyhow)?; - } - - // Set up async functions if we have a client - if let Some(ref state) = op_state_clone { - setup_async_ops(&ctx, &globals, state.clone())?; - } else { - // Set up stub functions that throw errors - setup_stub_functions(&ctx, &globals)?; - } - - // Set up results proxy if we have by_id context - if let Some(ref by_id) = by_id_clone { - setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?; - } - - // Determine if we need to add return statement. - // Wrap with .then((x) => JSON.stringify(x ?? null)) to serialize the result - // using the standard JSON.stringify, matching deno_core's behavior exactly. - let code = if should_add_return_quickjs(&transformed_expr) { - format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) - } else { - format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) - }; - - // Evaluate the expression (returns a Promise that resolves to a JSON string) - let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?; - - // Await the promise — result is already a JSON string from JSON.stringify - let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?; - - let json_str = String::from_js(&ctx, result) - .unwrap_or_else(|_| "null".to_string()); - - Ok(crate::common::unsafe_raw(json_str)) - }) - .await -} - -/// Set up async variable() and resource() functions using true Rust async callbacks. -/// -/// This uses rquickjs's `Async>` wrapper to create JavaScript functions that -/// return Promises. The Promises are resolved by spawned Rust async operations. -fn setup_async_ops<'js>( - ctx: &rquickjs::Ctx<'js>, - globals: &Object<'js>, - state: Arc, -) -> anyhow::Result<()> { - // Error prefix - must match the JavaScript side - const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; - - // Create variable() function with true async Rust callback - // Returns a JSON string that JavaScript will parse - let state_for_var = state.clone(); - globals.set( - "__fetchVariable", - Func::from(Async(MutFn::new(move |path: String| { - let client = state_for_var.client.clone(); - async move { - match client.get_variable_value(&path).await { - Ok(value) => value, - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create resource() function - returns JSON string - let state_for_res = state.clone(); - globals.set( - "__fetchResource", - Func::from(Async(MutFn::new(move |path: String| { - let client = state_for_res.client.clone(); - async move { - match client - .get_resource_value_interpolated::(&path, None) - .await - { - Ok(value) => { - serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) - } - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create JavaScript wrappers that parse the JSON results - // We use a unique prefix that's extremely unlikely to appear in real data - let wrapper_code = r#" - const __ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; - - async function variable(path) { - const result = await __fetchVariable(path); - if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { - throw new Error(result.substring(__ERR_PREFIX.length)); - } - return result; - } - - async function resource(path) { - const result = await __fetchResource(path); - if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { - throw new Error(result.substring(__ERR_PREFIX.length)); - } - return JSON.parse(result); - } - "#; - - ctx.eval::<(), _>(wrapper_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Set up stub functions that throw errors when no client is available -fn setup_stub_functions<'js>( - ctx: &rquickjs::Ctx<'js>, - _globals: &Object<'js>, -) -> anyhow::Result<()> { - let setup_code = r#" - function variable(path) { - return Promise.reject(new Error(`variable() is not available without an authenticated client`)); - } - - function resource(path) { - return Promise.reject(new Error(`resource() is not available without an authenticated client`)); - } - "#; - - ctx.eval::<(), _>(setup_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Set up the `results` Proxy object with dynamic access to step results. -/// -/// Uses async Rust callbacks to fetch results on-demand when accessed. -fn setup_results_proxy<'js>( - ctx: &rquickjs::Ctx<'js>, - globals: &Object<'js>, - by_id: &IdContext, - op_state: Option>, -) -> anyhow::Result<()> { - // Store previous_id for the shortcut optimization - globals.set("__previous_id", by_id.previous_id.clone())?; - - // Create async __getResult function that fetches step results via Rust - if let Some(state) = op_state { - let by_id_for_result = by_id.clone(); - globals.set( - "__fetchResult", - Func::from(Async(MutFn::new(move |step_id: String| { - let client = state.client.clone(); - let by_id = by_id_for_result.clone(); - let step_id_clone = step_id.clone(); - - // Look up the job ID(s) for this step from the local cache - let job_result = by_id.steps_results.get(&step_id).cloned(); - let flow_job_id = by_id.flow_job.to_string(); - - async move { - const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; - - let result: Result = match job_result { - Some(jr) => { - // Found in local cache, fetch result by job ID - match jr { - JobResult::SingleJob(job_id) => client - .get_completed_job_result::( - &job_id.to_string(), - None, - ) - .await - .map_err(|e| { - format!( - "Failed to fetch result for step '{}': {}", - step_id_clone, e - ) - }), - JobResult::ListJob(job_ids) => { - let futs = job_ids.iter().map(|job_id| { - let client = client.clone(); - let job_id_str = job_id.to_string(); - async move { - client - .get_completed_job_result::( - &job_id_str, - None, - ) - .await - } - }); - let results: Vec<_> = futures::future::join_all(futs).await; - let collected: Result, _> = - results.into_iter().collect(); - collected.map(serde_json::Value::Array).map_err(|e| { - format!( - "Failed to fetch results for step '{}': {}", - step_id_clone, e - ) - }) - } - } - } - None => { - // Not in local cache, fallback to querying by flow_job_id and step_id - // This happens for branch modules that need to access parent flow step results - // Use .ok() to match deno_core behavior: return null for non-existent steps - // instead of throwing an error - Ok(client - .get_result_by_id::( - &flow_job_id, - &step_id_clone, - None, - ) - .await - .ok() // Swallow errors, convert to Option - .unwrap_or(serde_json::Value::Null)) // None -> null - } - }; - - match result { - Ok(value) => { - serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) - } - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create JavaScript wrapper that parses the JSON result - let wrapper_code = r#" - const __RESULT_ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; - async function __getResult(stepId) { - const result = await __fetchResult(stepId); - if (typeof result === 'string' && result.startsWith(__RESULT_ERR_PREFIX)) { - throw new Error(result.substring(__RESULT_ERR_PREFIX.length)); - } - return JSON.parse(result); - } - "#; - ctx.eval::<(), _>(wrapper_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - } else { - // No client - stub function that rejects - let stub_code = r#" - function __getResult(stepId) { - return Promise.reject(new Error('Result fetching not available without authenticated client')); - } - "#; - ctx.eval::<(), _>(stub_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - } - - // Create the results proxy that calls __getResult for on-demand fetching - // Matches deno_core behavior: always try to fetch, let backend handle unknown step IDs - let proxy_setup = r#" - const results = new Proxy({}, { - get: function(target, name, receiver) { - // Handle symbol properties (like Symbol.toStringTag) - if (typeof name === 'symbol') { - return undefined; - } - - // Check if it's the previous_id and previous_result exists - if (name === __previous_id && typeof previous_result !== 'undefined') { - return Promise.resolve(previous_result); - } - - // Always try to fetch - let Rust/backend handle unknown step IDs - // This matches deno_core behavior - return __getResult(name); - } - }); - "#; - ctx.eval::<(), _>(proxy_setup) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Convert a serde_json::Value to a QuickJS Value -fn json_to_js<'js>( - ctx: &rquickjs::Ctx<'js>, - val: &serde_json::Value, -) -> rquickjs::Result> { - match val { - serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), - serde_json::Value::Bool(b) => Ok(Value::new_bool(ctx.clone(), *b)), - serde_json::Value::Number(n) => { - if let Some(i) = n.as_i64() { - if i >= i32::MIN as i64 && i <= i32::MAX as i64 { - Ok(Value::new_int(ctx.clone(), i as i32)) - } else { - Ok(Value::new_float(ctx.clone(), i as f64)) - } - } else if let Some(f) = n.as_f64() { - Ok(Value::new_float(ctx.clone(), f)) - } else { - Ok(Value::new_float(ctx.clone(), 0.0)) - } - } - serde_json::Value::String(s) => s.clone().into_js(ctx), - serde_json::Value::Array(arr) => { - let js_arr = rquickjs::Array::new(ctx.clone())?; - for (i, item) in arr.iter().enumerate() { - js_arr.set(i, json_to_js(ctx, item)?)?; - } - Ok(js_arr.into_value()) - } - serde_json::Value::Object(obj) => { - let js_obj = Object::new(ctx.clone())?; - for (k, v) in obj { - js_obj.set(k.as_str(), json_to_js(ctx, v)?)?; - } - Ok(js_obj.into_value()) - } - } -} - -/// Convert a QuickJS Value to a serde_json::Value -/// -/// This mimics JavaScript's JSON.stringify behavior: -/// - For objects with a `toJSON` method (like Date), call it and use the result -/// - Arrays are recursively serialized -/// - Plain objects enumerate their own properties - -/// Determines if we should prepend "return" to the expression -fn should_add_return_quickjs(expr: &str) -> bool { - let trimmed = expr.trim(); - - if trimmed.is_empty() { - return true; - } - - if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { - return false; - } - - let statement_prefixes = [ - "const ", - "let ", - "var ", - "if ", - "if(", - "for ", - "for(", - "while ", - "while(", - "switch ", - "switch(", - "try ", - "try{", - "throw ", - "function ", - "class ", - "async ", - "await ", - ]; - - for prefix in &statement_prefixes { - if trimmed.starts_with(prefix) { - return false; - } - } - - if contains_semicolon_outside_strings(trimmed) { - return false; - } - - true -} - -fn contains_semicolon_outside_strings(expr: &str) -> bool { - let mut in_single_quote = false; - let mut in_double_quote = false; - let mut in_template = false; - let mut prev_char = '\0'; - - for ch in expr.chars() { - match ch { - '\'' if prev_char != '\\' && !in_double_quote && !in_template => { - in_single_quote = !in_single_quote; - } - '"' if prev_char != '\\' && !in_single_quote && !in_template => { - in_double_quote = !in_double_quote; - } - '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { - in_template = !in_template; - } - ';' if !in_single_quote && !in_double_quote && !in_template => { - return true; - } - _ => {} - } - prev_char = ch; - } - - false -} - -fn quickjs_error_to_anyhow(err: rquickjs::CaughtError<'_>) -> anyhow::Error { - anyhow::anyhow!("QuickJS evaluation error: {}", err) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use windmill_common::worker::to_raw_value; - - #[tokio::test] - async fn test_eval_quickjs_simple() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); - - let result = - eval_timeout_quickjs("x + y".to_string(), env, None, None, None, None, None).await?; - - assert_eq!(result.get(), "8"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_object_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 42, "nested": {"value": 100}}))), - ); - - let result = eval_timeout_quickjs( - "params.test".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "42"); - - let result2 = eval_timeout_quickjs( - "params.nested.value".to_string(), - env, - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result2.get(), "100"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_array() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - let result = eval_timeout_quickjs( - "arr.map(x => x * 2)".to_string(), - env, - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "[2,4,6,8,10]"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_flow_input() -> anyhow::Result<()> { - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(10))); - - let result = eval_timeout_quickjs( - "flow_input.name".to_string(), - HashMap::new(), - Some(mappable_rc::Marc::new(flow_input)), - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "\"test\""); - Ok(()) - } - - #[test] - fn test_should_add_return_quickjs() { - assert!(should_add_return_quickjs("5")); - assert!(should_add_return_quickjs("x + y")); - assert!(should_add_return_quickjs("foo()")); - - assert!(!should_add_return_quickjs("return 5")); - assert!(!should_add_return_quickjs("return x + y")); - - assert!(!should_add_return_quickjs("const x = 5")); - assert!(!should_add_return_quickjs("let y = 10")); - assert!(!should_add_return_quickjs("if (x > 5) { return x; }")); - - assert!(!should_add_return_quickjs("let x = 5; x + 1")); - } - - #[tokio::test] - async fn test_eval_quickjs_date_serialization() -> anyhow::Result<()> { - // Test that Date objects serialize to ISO strings, matching JSON.stringify behavior - let result = eval_timeout_quickjs( - "new Date('2024-01-15T12:30:00.000Z')".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Should be an ISO string, not an empty object - assert_eq!(result.get(), "\"2024-01-15T12:30:00.000Z\""); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_date_in_object() -> anyhow::Result<()> { - // Test that Date objects within other objects serialize correctly - let result = eval_timeout_quickjs( - "({ date: new Date('2024-01-15T12:30:00.000Z'), name: 'test' })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value["date"], "2024-01-15T12:30:00.000Z"); - assert_eq!(value["name"], "test"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_date_in_array() -> anyhow::Result<()> { - // Test that Date objects in arrays serialize correctly - let result = eval_timeout_quickjs( - "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]" - .to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value[0], "2024-01-15T00:00:00.000Z"); - assert_eq!(value[1], "2024-01-16T00:00:00.000Z"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_custom_tojson() -> anyhow::Result<()> { - // Test that JSON.stringify handles custom toJSON when returning objects - let result = eval_timeout_quickjs( - r#"({ a: 1, toJSON: () => ({ converted: true }) })"#.to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // JSON.stringify should call toJSON and use that result - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value["converted"], true); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_deeply_nested_date() -> anyhow::Result<()> { - // Test that Date objects deep in nested structures are handled - let result = eval_timeout_quickjs( - "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!( - value["level1"]["level2"]["date"], - "2024-01-15T00:00:00.000Z" - ); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_regexp_serialization() -> anyhow::Result<()> { - // RegExp objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "/test/gi".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // RegExp doesn't have toJSON, so it serializes to an empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_map_serialization() -> anyhow::Result<()> { - // Map objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "new Map([['key', 'value']])".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Map doesn't have toJSON, serializes to empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_set_serialization() -> anyhow::Result<()> { - // Set objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "new Set([1, 2, 3])".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Set doesn't have toJSON, serializes to empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_tojson_returning_date() -> anyhow::Result<()> { - // When toJSON returns a Date object, JSON.stringify does NOT call toJSON again - // on the returned value (per spec). Date has no own enumerable properties, - // so it serializes to {}. - let result = eval_timeout_quickjs( - "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "{}"); - Ok(()) - } -} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index caf2b84ec3..741458c0e0 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -39,10 +39,6 @@ pub mod job_logger; pub mod job_logger_ee; mod job_logger_oss; mod js_eval; -#[cfg(test)] -mod js_eval_parity_tests; -#[cfg(feature = "quickjs")] -pub mod js_eval_quickjs; pub mod memory_common; #[cfg(feature = "private")] pub mod memory_ee; diff --git a/backend/windmill-worker/src/runtime.js b/backend/windmill-worker/src/runtime.js deleted file mode 100644 index 0feafb59ec..0000000000 --- a/backend/windmill-worker/src/runtime.js +++ /dev/null @@ -1,212 +0,0 @@ -import * as abortSignal from "ext:deno_web/03_abort_signal.js"; -import * as base64 from "ext:deno_web/05_base64.js"; -import * as console from "ext:deno_console/01_console.js"; -import * as encoding from "ext:deno_web/08_text_encoding.js"; -import * as event from "ext:deno_web/02_event.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; -import * as file from "ext:deno_web/09_file.js"; -import * as fileReader from "ext:deno_web/10_filereader.js"; -import * as formData from "ext:deno_fetch/21_formdata.js"; -import * as headers from "ext:deno_fetch/20_headers.js"; -import * as streams from "ext:deno_web/06_streams.js"; -import * as timers from "ext:deno_web/02_timers.js"; -import * as url from "ext:deno_url/00_url.js"; -import * as net from "ext:deno_net/01_net.js"; -import * as tls from "ext:deno_net/02_tls.js"; -import * as urlPattern from "ext:deno_url/01_urlpattern.js"; -import * as webidl from "ext:deno_webidl/00_webidl.js"; -import * as response from "ext:deno_fetch/23_response.js"; -import * as request from "ext:deno_fetch/23_request.js"; -import "ext:deno_web/02_structured_clone.js"; -import "ext:deno_web/04_global_interfaces.js"; -import "ext:deno_web/13_message_port.js"; -import "ext:deno_web/14_compression.js"; -import "ext:deno_web/15_performance.js"; -import "ext:deno_web/16_image_data.js"; -import "ext:deno_fetch/27_eventsource.js"; - -globalThis.atob = base64.atob; -globalThis.btoa = base64.btoa; -globalThis.fetch = fetch.fetch; -globalThis.Request = request.Request; -globalThis.Response = response.Response; -globalThis.Blob = file.Blob; -globalThis.URL = url.URL; -globalThis.FormData = formData.FormData; -globalThis.URLSearchParams = url.URLSearchParams; -globalThis.Headers = headers.Headers; -globalThis.FileReader = fileReader.FileReader; -globalThis.console = new console.Console((msg, level) => - globalThis.Deno.core.ops.op_log(msg) -); -globalThis.AbortController = abortSignal.AbortController; -globalThis.AbortSignal = abortSignal.AbortSignal; - -Object.assign(globalThis, { - clearInterval: timers.clearInterval, - clearTimeout: timers.clearTimeout, - setInterval: timers.setInterval, - setTimeout: timers.setTimeout, -}); - -// Expose bootstrapOtel globally so it can be called from Rust after runtime creation. -// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. -// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] -// consoleConfig: 0=ignore, 1=capture, 2=replace -globalThis.__bootstrapOtel = () => { - import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { - bootstrap([1, 0, 1, 0]); - // Expose enterSpan for setting parent trace context - globalThis.__enterSpan = enterSpan; - }); -}; - -// Object.assign(globalThis, { -// console: nonEnumerable( -// new console.Console((msg, level) => core.print(msg, level > 1)) -// ), - -// // timers - -// // fetch -// Request: nonEnumerable(request.Request), -// Response: nonEnumerable(response.Response), -// Headers: nonEnumerable(headers.Headers), -// fetch: writable(fetch.fetch), - -// // base64 -// atob: writable(base64.atob), -// btoa: writable(base64.btoa), - -// // encoding -// TextDecoder: nonEnumerable(encoding.TextDecoder), -// TextEncoder: nonEnumerable(encoding.TextEncoder), -// TextDecoderStream: nonEnumerable(encoding.TextDecoderStream), -// TextEncoderStream: nonEnumerable(encoding.TextEncoderStream), - -// // url -// URL: nonEnumerable(url.URL), -// URLPattern: nonEnumerable(urlPattern.URLPattern), -// URLSearchParams: nonEnumerable(url.URLSearchParams), - -// // // crypto -// // CryptoKey: nonEnumerable(crypto.CryptoKey), -// // crypto: readOnly(crypto.crypto), -// // Crypto: nonEnumerable(crypto.Crypto), -// // SubtleCrypto: nonEnumerable(crypto.SubtleCrypto), - -// // streams -// ByteLengthQueuingStrategy: nonEnumerable(streams.ByteLengthQueuingStrategy), -// CountQueuingStrategy: nonEnumerable(streams.CountQueuingStrategy), -// ReadableStream: nonEnumerable(streams.ReadableStream), -// ReadableStreamDefaultReader: nonEnumerable( -// streams.ReadableStreamDefaultReader -// ), -// ReadableByteStreamController: nonEnumerable( -// streams.ReadableByteStreamController -// ), -// ReadableStreamBYOBReader: nonEnumerable(streams.ReadableStreamBYOBReader), -// ReadableStreamBYOBRequest: nonEnumerable(streams.ReadableStreamBYOBRequest), -// ReadableStreamDefaultController: nonEnumerable( -// streams.ReadableStreamDefaultController -// ), -// TransformStream: nonEnumerable(streams.TransformStream), -// TransformStreamDefaultController: nonEnumerable( -// streams.TransformStreamDefaultController -// ), -// WritableStream: nonEnumerable(streams.WritableStream), -// WritableStreamDefaultWriter: nonEnumerable( -// streams.WritableStreamDefaultWriter -// ), -// WritableStreamDefaultController: nonEnumerable( -// streams.WritableStreamDefaultController -// ), - -// // event -// CloseEvent: nonEnumerable(event.CloseEvent), -// CustomEvent: nonEnumerable(event.CustomEvent), -// ErrorEvent: nonEnumerable(event.ErrorEvent), -// Event: nonEnumerable(event.Event), -// EventTarget: nonEnumerable(event.EventTarget), -// MessageEvent: nonEnumerable(event.MessageEvent), -// PromiseRejectionEvent: nonEnumerable(event.PromiseRejectionEvent), -// ProgressEvent: nonEnumerable(event.ProgressEvent), -// reportError: writable(event.reportError), -// DOMException: nonEnumerable(DOMException), - -// // file -// Blob: nonEnumerable(file.Blob), -// File: nonEnumerable(file.File), -// FileReader: nonEnumerable(fileReader.FileReader), - -// // form data -// FormData: nonEnumerable(formData.FormData), - -// // abort signal -// AbortController: nonEnumerable(abortSignal.AbortController), -// AbortSignal: nonEnumerable(abortSignal.AbortSignal), - -// // // web sockets -// // WebSocket: nonEnumerable(webSocket.WebSocket), - -// // // performance -// // Performance: nonEnumerable(performance.Performance), -// // PerformanceEntry: nonEnumerable(performance.PerformanceEntry), -// // PerformanceMark: nonEnumerable(performance.PerformanceMark), -// // PerformanceMeasure: nonEnumerable(performance.PerformanceMeasure), -// // performance: writable(performance.performance), - -// // messagePort -// // structuredClone: writable(messagePort.structuredClone), - -// // Branding as a WebIDL object -// [webidl.brand]: nonEnumerable(webidl.brand), -// }); - -// function nonEnumerable(value) { -// return { -// value, -// writable: true, -// enumerable: false, -// configurable: true, -// }; -// } - -// function writable(value) { -// return { -// value, -// writable: true, -// enumerable: true, -// configurable: true, -// }; -// } - -// function readOnly(value) { -// return { -// value, -// enumerable: true, -// writable: false, -// configurable: true, -// }; -// } - -// function getterOnly(getter) { -// return { -// get: getter, -// set() {}, -// enumerable: true, -// configurable: true, -// }; -// } - -// function formatException(error) { -// if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) { -// return null; -// } else if (typeof error == "string") { -// return `Uncaught ${console.inspectArgs([console.quoteString(error)], { -// colors: false, -// })}`; -// } else { -// return `Uncaught ${console.inspectArgs([error], { colors: false })}`; -// } -// }