refactor: isolate deno_core into windmill-runtime-nativets subcrate (#7848)

* refactor: isolate deno_core into windmill-runtime-nativets subcrate

Remove deno_core from flow eval and isolate nativets V8 runtime into a
dedicated subcrate so deno_core compilation no longer blocks
windmill-worker or windmill-api.

- Create windmill-jseval crate: QuickJS-based JS eval for flow
  expressions and batch rerun, extracted from windmill-worker
- Create windmill-runtime-nativets crate: all deno_core/V8 deps and
  nativets script execution, with build.rs snapshot generation
- Simplify windmill-worker: remove all deno_* direct deps, empty
  build.rs, gate nativets behind optional dep
- Update windmill-api: use windmill-jseval for batch rerun instead of
  deno_core, remove deno_core feature entirely
- Add nativets integration tests (nativets_jobs.rs) and parallel
  stress test (nativets_stress.rs, 8 workers x 200 jobs)
- Remove dead code: deno flow eval path, USE_QUICKJS env var,
  parity tests (replaced with 63 standalone expected-value tests)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: address PR review feedback for deno_core isolation

- Deduplicate unsafe_raw() into windmill-common/src/utils.rs (single source)
- Delete orphaned runtime.js and windmill-client.js from windmill-worker/src/
- Fix operator precedence in windmill-jseval with explicit parentheses
- Remove unnecessary return keyword in heap limit callback
- Remove redundant as usize casts
- Remove ~150 lines of commented-out code from runtime.js
- Remove commented-out #[cfg] in build.rs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* otel ee

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-02-08 10:16:37 +01:00
committed by GitHub
parent f22b62d838
commit 3f68e3a58e
24 changed files with 4450 additions and 7880 deletions

79
backend/Cargo.lock generated
View File

@@ -15663,7 +15663,6 @@ dependencies = [
"bitflags 2.9.4",
"chrono",
"constant_time_eq 0.3.1",
"deno_core",
"dotenv",
"futures",
"gethostname",
@@ -15702,7 +15701,6 @@ dependencies = [
"tracing",
"url",
"uuid",
"v8",
"windmill-api",
"windmill-api-client",
"windmill-autoscaling",
@@ -15711,6 +15709,7 @@ dependencies = [
"windmill-git-sync",
"windmill-indexer",
"windmill-queue",
"windmill-runtime-nativets",
"windmill-worker",
"windows-service",
"windows-sys 0.52.0",
@@ -15745,8 +15744,6 @@ dependencies = [
"cron",
"dashmap 6.1.0",
"datafusion",
"deno_core",
"deno_error",
"ed25519-dalek",
"flate2",
"futures",
@@ -15814,6 +15811,7 @@ dependencies = [
"windmill-dep-map",
"windmill-git-sync",
"windmill-indexer",
"windmill-jseval",
"windmill-mcp",
"windmill-native-triggers",
"windmill-oauth",
@@ -16122,6 +16120,23 @@ dependencies = [
"windmill-common",
]
[[package]]
name = "windmill-jseval"
version = "1.628.3"
dependencies = [
"anyhow",
"futures",
"lazy_static",
"mappable-rc",
"regex",
"rquickjs",
"serde_json",
"tokio",
"tracing",
"uuid",
"windmill-common",
]
[[package]]
name = "windmill-macros"
version = "1.628.3"
@@ -16454,6 +16469,43 @@ dependencies = [
"windmill-common",
]
[[package]]
name = "windmill-runtime-nativets"
version = "1.628.3"
dependencies = [
"anyhow",
"const_format",
"deno_ast",
"deno_console",
"deno_core",
"deno_error",
"deno_fetch",
"deno_io",
"deno_net",
"deno_permissions",
"deno_runtime",
"deno_telemetry",
"deno_tls",
"deno_url",
"deno_web",
"deno_webidl",
"futures",
"itertools 0.14.0",
"lazy_static",
"regex",
"reqwest 0.13.1",
"serde",
"serde_json",
"sqlx",
"tokio",
"tracing",
"uuid",
"winapi",
"windmill-common",
"windmill-parser-ts",
"windmill-queue",
]
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.628.3"
@@ -16792,20 +16844,6 @@ dependencies = [
"chrono",
"const_format",
"convert_case 0.6.0",
"deno_ast",
"deno_console",
"deno_core",
"deno_error",
"deno_fetch",
"deno_io",
"deno_net",
"deno_permissions",
"deno_runtime",
"deno_telemetry",
"deno_tls",
"deno_url",
"deno_web",
"deno_webidl",
"derive_more 1.0.0",
"dotenv",
"eventsource-stream",
@@ -16844,9 +16882,7 @@ dependencies = [
"regex",
"reqwest 0.13.1",
"reqwest-middleware",
"rquickjs",
"rust_decimal",
"rustls-pemfile 2.2.0",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -16863,11 +16899,11 @@ dependencies = [
"url",
"urlencoding",
"uuid",
"winapi",
"windmill-audit",
"windmill-common",
"windmill-dep-map",
"windmill-git-sync",
"windmill-jseval",
"windmill-macros",
"windmill-mcp",
"windmill-parser",
@@ -16886,6 +16922,7 @@ dependencies = [
"windmill-parser-ts",
"windmill-parser-yaml",
"windmill-queue",
"windmill-runtime-nativets",
"yaml-rust",
]

View File

@@ -28,6 +28,8 @@ members = [
"./windmill-worker",
"./windmill-dep-map",
"./windmill-common",
"./windmill-jseval",
"./windmill-runtime-nativets",
"./windmill-mcp",
"./windmill-audit",
"./windmill-git-sync",
@@ -85,8 +87,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"]
quickjs = ["windmill-worker/quickjs"]
deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets"]
deno_core_mac = ["deno_core", "windmill-worker/libffi_mac"]
kafka = ["windmill-api/kafka"]
kafka-gssapi = ["windmill-api/kafka-gssapi"]
@@ -193,13 +194,12 @@ serde_json.workspace = true
serde_derive.workspace = true
serde_yml.workspace = true
serde.workspace = true
deno_core = { workspace = true, optional = true }
windmill-runtime-nativets = { workspace = true, optional = true }
object_store = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
constant_time_eq = { workspace = true, optional = true }
quote.workspace = true
memchr.workspace = true
v8 = { workspace = true, optional = true }
rustls.workspace = true
pep440_rs.workspace = true
strum.workspace = true
@@ -230,7 +230,6 @@ windmill-dep-map.workspace = true
axum.workspace = true
serde.workspace = true
windmill-api-client.workspace = true
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] }
tempfile.workspace = true
@@ -278,6 +277,8 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-jseval = { path = "./windmill-jseval" }
windmill-runtime-nativets = { path = "./windmill-runtime-nativets" }
windmill-api-client = { path = "./windmill-api-client" }
reqwest-retry = "^0"

View File

@@ -1 +1 @@
a69c11d8279401ede1ad5b54e3678c3efb3d2381
327cf1bff1c5a61f6ea2bd81f1476bee51d152c5

View File

@@ -127,27 +127,8 @@ mod monitor;
mod windows_service_ee;
pub fn setup_deno_runtime() -> anyhow::Result<()> {
// https://github.com/denoland/deno/blob/main/cli/main.rs#L477
#[cfg(feature = "deno_core")]
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
"--stack-size=1024".to_string(),
// TODO(bartlomieju): I think this can be removed as it's handled by `deno_core`
// and its settings.
// deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript
// TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast
"--no-harmony-import-assertions".to_string(),
])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
#[cfg(feature = "deno_core")]
if !unrecognized_v8_flags.is_empty() {
println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags);
}
#[cfg(feature = "deno_core")]
deno_core::JsRuntime::init_platform(None, false);
windmill_runtime_nativets::setup_deno_runtime()?;
Ok(())
}

View File

@@ -0,0 +1,279 @@
/*
* Integration tests for nativets (//native) job execution.
*
* These tests ensure that the V8 runtime is properly initialized and that
* nativets jobs execute correctly without segfaults. They cover sync functions,
* async functions, and fetch operations.
*
* All nativets tests run inside a single #[sqlx::test] with a single long-lived
* worker, because V8 isolates cannot be safely created/destroyed/recreated
* across different tokio runtimes or worker lifecycles.
*
* Run with:
* cargo test -p windmill --features "deno_core" --test nativets_jobs -- --nocapture
*/
#[cfg(feature = "deno_core")]
mod common;
#[cfg(feature = "deno_core")]
use common::*;
#[cfg(feature = "deno_core")]
use futures::StreamExt;
#[cfg(feature = "deno_core")]
use sqlx::{Pool, Postgres};
#[cfg(feature = "deno_core")]
use windmill_common::jobs::{JobPayload, RawCode};
#[cfg(feature = "deno_core")]
use windmill_common::scripts::ScriptLang;
#[cfg(feature = "deno_core")]
fn init_nativets_runtime() {
static INIT: std::sync::Once = std::sync::Once::new();
INIT.call_once(|| {
let _ = rustls::crypto::ring::default_provider().install_default();
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
});
}
#[cfg(feature = "deno_core")]
fn nativets_code(content: &str) -> JobPayload {
JobPayload::Code(RawCode {
hash: None,
content: content.to_string(),
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
})
}
#[cfg(feature = "deno_core")]
async fn push_and_wait(
db: &Pool<Postgres>,
job: RunJob,
listener: &mut (impl StreamExt<Item = uuid::Uuid> + Unpin),
) -> CompletedJob {
let uuid = job.push(db).await;
let deadline = std::time::Instant::now() + tokio::time::Duration::from_secs(60);
loop {
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
if remaining.is_zero() {
panic!("Timed out waiting for job {uuid}");
}
match tokio::time::timeout(remaining, listener.next()).await {
Ok(Some(completed_uuid)) if completed_uuid == uuid => break,
Ok(Some(_)) => continue,
Ok(None) => panic!("Listener ended while waiting for {uuid}"),
Err(_) => panic!("Timed out waiting for job {uuid}"),
}
}
completed_job(uuid, db).await
}
/// All nativets tests share a single worker to avoid V8 isolate lifecycle issues.
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_nativets_jobs(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
init_nativets_runtime();
set_jwt_secret().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let mut listener = listen_for_completed_jobs(&db).await;
let conn = windmill_common::worker::Connection::Sql(db.clone());
let (killpill, _worker_handle) = spawn_test_worker(&conn, port);
// -- simple string return --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(): string {
return "hello from nativets";
}
"#,
)),
&mut listener,
)
.await;
assert!(result.success, "simple_return failed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap(),
serde_json::json!("hello from nativets")
);
}
// -- arithmetic with args --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(x: number, y: number): number {
return x + y;
}
"#,
))
.arg("x", serde_json::json!(17))
.arg("y", serde_json::json!(25)),
&mut listener,
)
.await;
assert!(result.success, "with_args failed: {:?}", result.result);
assert_eq!(result.json_result().unwrap(), serde_json::json!(42));
}
// -- complex object return --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(items: {name: string, value: number}[]): {total: number, names: string[]} {
return {
total: items.reduce((sum, i) => sum + i.value, 0),
names: items.map(i => i.name).sort(),
};
}
"#,
))
.arg(
"items",
serde_json::json!([
{"name": "c", "value": 30},
{"name": "a", "value": 10},
{"name": "b", "value": 20}
]),
),
&mut listener,
)
.await;
assert!(result.success, "object_return failed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap(),
serde_json::json!({"total": 60, "names": ["a", "b", "c"]})
);
}
// -- async function (Promise) --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export async function main(): Promise<number> {
const result = await Promise.resolve(42);
return result;
}
"#,
)),
&mut listener,
)
.await;
assert!(result.success, "async failed: {:?}", result.result);
assert_eq!(result.json_result().unwrap(), serde_json::json!(42));
}
// -- fetch hitting the internal API server --
{
let code = format!(
r#"//native
export async function main(): Promise<object> {{
const resp = await fetch("http://localhost:{port}/api/version");
return {{ status: resp.status, ok: resp.ok }};
}}
"#
);
let result = push_and_wait(&db, RunJob::from(nativets_code(&code)), &mut listener).await;
assert!(result.success, "fetch failed: {:?}", result.result);
let val = result.json_result().unwrap();
assert_eq!(val["ok"], serde_json::json!(true));
assert_eq!(val["status"], serde_json::json!(200));
}
// -- regex --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(text: string, pattern: string): string[] {
const regex = new RegExp(pattern, 'g');
return [...text.matchAll(regex)].map(m => m[0]);
}
"#,
))
.arg("text", serde_json::json!("foo123bar456baz789"))
.arg("pattern", serde_json::json!("\\d+")),
&mut listener,
)
.await;
assert!(result.success, "regex failed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap(),
serde_json::json!(["123", "456", "789"])
);
}
// -- JSON roundtrip --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(): string {
const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } });
const parsed = JSON.parse(data);
return parsed.nested.arr.map((x: number) => x * 10).join(",");
}
"#,
)),
&mut listener,
)
.await;
assert!(result.success, "json_roundtrip failed: {:?}", result.result);
assert_eq!(result.json_result().unwrap(), serde_json::json!("10,20,30"));
}
// -- no args, array return --
{
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(): number[] {
return Array.from({length: 5}, (_, i) => i * i);
}
"#,
)),
&mut listener,
)
.await;
assert!(result.success, "no_args failed: {:?}", result.result);
assert_eq!(
result.json_result().unwrap(),
serde_json::json!([0, 1, 4, 9, 16])
);
}
killpill.send();
Ok(())
}

View File

@@ -0,0 +1,428 @@
/*
* Stress test for parallel nativets execution.
*
* Spawns 8 workers and pushes hundreds of fast nativets jobs concurrently.
* Tests: arithmetic, JSON manipulation, string ops, fetch to an internal HTTP server,
* and async operations. Validates all jobs complete successfully with correct results.
*
* IMPORTANT: V8 segfaults when the test harness captures stdout (default behavior).
* Always run with --nocapture:
* cargo test -p windmill --features "deno_core" --test nativets_stress -- --nocapture
*/
#[cfg(feature = "deno_core")]
mod common;
#[cfg(feature = "deno_core")]
use common::*;
#[cfg(feature = "deno_core")]
use std::time::Instant;
#[cfg(feature = "deno_core")]
use futures::StreamExt;
#[cfg(feature = "deno_core")]
use serde_json::json;
#[cfg(feature = "deno_core")]
use sqlx::{Pool, Postgres};
#[cfg(feature = "deno_core")]
use uuid::Uuid;
#[cfg(feature = "deno_core")]
use windmill_common::{
jobs::{JobPayload, RawCode},
scripts::ScriptLang,
worker::{Connection, WORKER_CONFIG},
KillpillSender,
};
#[cfg(feature = "deno_core")]
use windmill_queue::PushIsolationLevel;
#[cfg(feature = "deno_core")]
const NUM_WORKERS: usize = 8;
/// Various nativets scripts that are fast to execute but cover different features.
#[cfg(feature = "deno_core")]
fn job_scripts() -> Vec<(&'static str, serde_json::Value, serde_json::Value)> {
vec![
// (script_code, args_json, expected_result)
(
r#"//native
export function main(x: number, y: number): number {
return x + y;
}
"#,
json!({"x": 10, "y": 32}),
json!(42),
),
(
r#"//native
export function main(s: string): string {
return s.split('').reverse().join('');
}
"#,
json!({"s": "hello"}),
json!("olleh"),
),
(
r#"//native
export function main(n: number): number[] {
return Array.from({length: n}, (_, i) => i * i);
}
"#,
json!({"n": 5}),
json!([0, 1, 4, 9, 16]),
),
(
r#"//native
export function main(items: {name: string, value: number}[]): {total: number, names: string[]} {
return {
total: items.reduce((sum, i) => sum + i.value, 0),
names: items.map(i => i.name).sort(),
};
}
"#,
json!({"items": [{"name": "c", "value": 30}, {"name": "a", "value": 10}, {"name": "b", "value": 20}]}),
json!({"total": 60, "names": ["a", "b", "c"]}),
),
(
r#"//native
export function main(a: number): object {
const fib = (n: number): number => n <= 1 ? n : fib(n - 1) + fib(n - 2);
return { input: a, fib: fib(a) };
}
"#,
json!({"a": 10}),
json!({"input": 10, "fib": 55}),
),
(
r#"//native
export function main(text: string, pattern: string): string[] {
const regex = new RegExp(pattern, 'g');
return [...text.matchAll(regex)].map(m => m[0]);
}
"#,
json!({"text": "foo123bar456baz789", "pattern": "\\d+"}),
json!(["123", "456", "789"]),
),
(
r#"//native
export function main(obj: Record<string, number>): Record<string, string> {
return Object.fromEntries(
Object.entries(obj).map(([k, v]) => [k.toUpperCase(), String(v * 2)])
);
}
"#,
json!({"obj": {"a": 1, "b": 2, "c": 3}}),
json!({"A": "2", "B": "4", "C": "6"}),
),
(
r#"//native
export function main(): string {
const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } });
const parsed = JSON.parse(data);
return parsed.nested.arr.map((x: number) => x * 10).join(",");
}
"#,
json!({}),
json!("10,20,30"),
),
]
}
#[cfg(feature = "deno_core")]
async fn push_job(db: &Pool<Postgres>, content: &str, args: &serde_json::Value) -> Uuid {
let mut hm_args = std::collections::HashMap::new();
if let Some(obj) = args.as_object() {
for (k, v) in obj {
hm_args.insert(k.clone(), windmill_common::worker::to_raw_value(v));
}
}
let job = JobPayload::Code(RawCode {
hash: None,
content: content.to_string(),
path: None,
language: ScriptLang::Bun, // //native annotation → nativets
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
});
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let (uuid, tx) = windmill_queue::push(
db,
tx,
"test-workspace",
job,
windmill_queue::PushArgs::from(&hm_args),
/* user */ "test-user",
/* email */ "test@windmill.dev",
/* permissioned_as */ "u/test-user".to_string(),
/* token_prefix */ None,
/* scheduled_for */ None,
/* schedule_path */ None,
/* parent_job */ None,
/* root_job */ None,
/* flow_innermost_root_job */ None,
/* job_id */ None,
/* is_flow_step */ false,
/* same_worker */ false,
None,
true,
None,
None,
None,
None,
None,
false,
None,
None,
None,
)
.await
.expect("push must succeed");
tx.commit().await.unwrap();
uuid
}
#[cfg(feature = "deno_core")]
fn spawn_workers(
conn: &Connection,
port: u16,
n: usize,
) -> (KillpillSender, Vec<tokio::task::JoinHandle<()>>) {
use std::sync::atomic::{AtomicUsize, Ordering};
static WORKER_ID: AtomicUsize = AtomicUsize::new(0);
std::fs::DirBuilder::new()
.recursive(true)
.create(windmill_worker::GO_BIN_CACHE_DIR)
.expect("could not create initial worker dir");
let (tx, _) = KillpillSender::new(n + 1);
let mut handles = Vec::with_capacity(n);
for i in 0..n {
let rx = tx.subscribe();
let conn = conn.clone();
let tx2 = tx.clone();
let id = WORKER_ID.fetch_add(1, Ordering::SeqCst);
let worker_name = format!("{id}/stress-w{i}");
let future = async move {
let base_internal_url = format!("http://localhost:{}", port);
{
let mut wc = WORKER_CONFIG.write().await;
wc.worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
wc.priority_tags_sorted = vec![windmill_common::worker::PriorityTags {
priority: 0,
tags: wc.worker_tags.clone(),
}];
windmill_common::worker::store_suspended_pull_query(&wc).await;
windmill_common::worker::store_pull_query(&wc).await;
}
windmill_worker::run_worker(
&conn,
"test-host",
worker_name,
i as u64,
n as u32,
"127.0.0.1",
rx,
tx2,
&base_internal_url,
)
.await;
};
handles.push(tokio::task::spawn(future));
}
(tx, handles)
}
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_parallel_nativets_stress(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
set_jwt_secret().await;
// V8 and rustls must be initialized before any JsRuntime is created
static RUNTIME_INIT: std::sync::Once = std::sync::Once::new();
RUNTIME_INIT.call_once(|| {
let _ = rustls::crypto::ring::default_provider().install_default();
windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed");
});
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let conn = Connection::Sql(db.clone());
// Set up completed job listener
let mut listener = listen_for_completed_jobs(&db).await;
// Prepare job scripts
let scripts = job_scripts();
// Also prepare fetch-based scripts that hit the internal API server
let fetch_script_template = |port: u16| -> Vec<(String, serde_json::Value)> {
vec![
(
format!(
r#"//native
export async function main(): Promise<object> {{
const resp = await fetch("http://localhost:{port}/api/version");
return {{ status: resp.status, ok: resp.ok }};
}}
"#
),
json!({}),
),
(
format!(
r#"//native
export async function main(): Promise<string> {{
const resp = await fetch("http://localhost:{port}/api/version");
const text = await resp.text();
return typeof text;
}}
"#
),
json!({}),
),
]
};
let fetch_scripts = fetch_script_template(port);
let total_jobs = 200;
let start = Instant::now();
let mut expected_results: Vec<(Uuid, Option<serde_json::Value>)> =
Vec::with_capacity(total_jobs);
for i in 0..total_jobs {
let (uuid, expected) = if i % 10 < 8 {
// 80% non-fetch jobs
let idx = i % scripts.len();
let (code, args, expected) = &scripts[idx];
let uuid = push_job(&db, code, args).await;
(uuid, Some(expected.clone()))
} else {
// 20% fetch jobs (we don't check exact result since version string varies)
let idx = i % fetch_scripts.len();
let (code, args) = &fetch_scripts[idx];
let uuid = push_job(&db, code, args).await;
(uuid, None) // Don't check exact value, just success
};
expected_results.push((uuid, expected));
}
let push_duration = start.elapsed();
tracing::info!(
"Pushed {} jobs in {:?} ({:?}/job)",
total_jobs,
push_duration,
push_duration / total_jobs as u32
);
// Spawn 8 workers
let (killpill, worker_handles) = spawn_workers(&conn, port, NUM_WORKERS);
// Wait for all jobs to complete
let mut completed: std::collections::HashSet<Uuid> = std::collections::HashSet::new();
let timeout_dur = tokio::time::Duration::from_secs(120);
let deadline = Instant::now() + timeout_dur;
while completed.len() < total_jobs {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
panic!(
"Timed out waiting for jobs: {}/{} completed",
completed.len(),
total_jobs
);
}
match tokio::time::timeout(remaining, listener.next()).await {
Ok(Some(uuid)) => {
completed.insert(uuid);
if completed.len() % 50 == 0 {
tracing::info!(
"Progress: {}/{} jobs completed",
completed.len(),
total_jobs
);
}
}
Ok(None) => panic!("Listener stream ended"),
Err(_) => panic!("Timed out: {}/{} completed", completed.len(), total_jobs),
}
}
let exec_duration = start.elapsed();
tracing::info!(
"All {} jobs completed in {:?} with {} workers ({:?}/job avg)",
total_jobs,
exec_duration,
NUM_WORKERS,
exec_duration / total_jobs as u32
);
// Kill workers
killpill.send();
for handle in worker_handles {
let _ = tokio::time::timeout(std::time::Duration::from_secs(10), handle).await;
}
// Verify results
let mut successes = 0;
let mut failures = 0;
for (uuid, expected) in &expected_results {
let job = completed_job(*uuid, &db).await;
if !job.success {
failures += 1;
tracing::error!("Job {} FAILED: {:?}", uuid, job.result);
continue;
}
successes += 1;
if let Some(expected_val) = expected {
let result = job
.json_result()
.expect("successful job should have result");
assert_eq!(
result, *expected_val,
"Job {} produced wrong result.\nExpected: {}\nGot: {}",
uuid, expected_val, result
);
}
}
tracing::info!(
"Results: {} successes, {} failures out of {} total",
successes,
failures,
total_jobs
);
assert_eq!(failures, 0, "All jobs should succeed");
assert_eq!(successes, total_jobs, "All jobs should be verified");
Ok(())
}

View File

@@ -36,7 +36,6 @@ postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-store/postgres_tr
mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-store/mqtt_trigger"]
native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "dep:strum", "oauth2"]
sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-store/sqs_trigger"]
deno_core = ["dep:deno_core", "dep:deno_error"]
gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-store/gcp_trigger"]
cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud"]
mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"]
@@ -155,12 +154,10 @@ aws-sdk-bedrock = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { workspace = true, optional = true }
aws-smithy-types.workspace = true
async-trait.workspace = true
deno_error = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
windmill-jseval.workspace = true
tar.workspace = true
flate2.workspace = true
strum = { workspace = true, optional = true }
dashmap.workspace = true
[build-dependencies]
deno_core = { workspace = true, optional = true }

View File

@@ -12,8 +12,6 @@ pub use windmill_api_jobs::types::*;
pub use windmill_api_sse::*;
use axum::body::Body;
#[cfg(feature = "deno_core")]
use deno_core::{op2, serde_v8, v8, JsRuntime, OpState};
use futures::{StreamExt, TryFutureExt};
use itertools::Itertools;
use quick_cache::sync::Cache;
@@ -2928,50 +2926,18 @@ struct BatchReRunQueryReturnType {
schema: Option<serde_json::Value>,
}
#[cfg(feature = "deno_core")]
#[op2]
#[string]
fn get_deno_core_job_value(state: &mut OpState) -> Option<String> {
let obj = state.borrow::<BatchReRunQueryReturnType>();
let str = serde_json::to_string(&obj).ok()?;
Some(str)
}
#[cfg(feature = "deno_core")]
async fn batch_rerun_compute_js_expression(
expr: String,
job: BatchReRunQueryReturnType,
) -> error::Result<Box<RawValue>> {
let ext = deno_core::Extension {
name: "batch_rerun_arg_transform_ext",
ops: vec![get_deno_core_job_value()].into(),
..Default::default()
};
let mut isolate =
JsRuntime::new(deno_core::RuntimeOptions { extensions: vec![ext], ..Default::default() });
{
let op_state = isolate.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(BatchReRunQueryReturnType { schema: None, ..job });
}
isolate
.execute_script(
"<batch_rerun_arg_transform>",
"let job = JSON.parse(Deno.core.ops.get_deno_core_job_value());",
)
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
// Run user expr
let result = isolate
.execute_script("<batch_rerun_arg_transform>", expr)
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
let mut scope = isolate.handle_scope();
let result = v8::Local::new(&mut scope, result);
let result: serde_json::Value =
serde_v8::from_v8(&mut scope, result).map_err(|e| Error::ExecutionErr(e.to_string()))?;
let result = JsonRawValue::from_string(result.to_string())?;
Ok(result)
let job_no_schema = BatchReRunQueryReturnType { schema: None, ..job };
let job_value =
serde_json::to_value(&job_no_schema).map_err(|e| Error::ExecutionErr(e.to_string()))?;
let mut globals = std::collections::HashMap::new();
globals.insert("job".to_string(), job_value);
windmill_jseval::eval_simple_js(expr, globals)
.await
.map_err(|e| Error::ExecutionErr(e.to_string()))
}
async fn batch_rerun_jobs(
@@ -3098,13 +3064,6 @@ async fn batch_rerun_handle_job(
args.insert(property_name.clone(), value.clone());
}
InputTransform::Javascript { expr } => {
#[cfg(not(feature = "deno_core"))]
Err(error::Error::ExecutionErr(
format!("deno_core feature is not activated, cannot evaluate: {expr}")
.to_string(),
))?;
#[cfg(feature = "deno_core")]
args.insert(
property_name.clone(),
batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?,

View File

@@ -974,6 +974,14 @@ pub async fn get_custom_pg_instance_password(db: &DB) -> Result<String> {
)
}
/// Convert a JSON string to a `Box<RawValue>` without validation.
///
/// # Safety
/// The caller must ensure the string is valid JSON.
pub fn unsafe_raw(json: String) -> Box<serde_json::value::RawValue> {
unsafe { std::mem::transmute::<Box<str>, Box<serde_json::value::RawValue>>(json.into()) }
}
// Avoid JSON parsing for merging raw JSON values into an object
pub fn merge_raw_values_to_object(
pairs: &[(String, Box<serde_json::value::RawValue>)],

View File

@@ -0,0 +1,25 @@
[package]
name = "windmill-jseval"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_jseval"
path = "src/lib.rs"
[features]
default = []
[dependencies]
windmill-common = { workspace = true, default-features = false }
rquickjs.workspace = true
serde_json.workspace = true
tokio.workspace = true
tracing.workspace = true
anyhow.workspace = true
regex.workspace = true
lazy_static.workspace = true
mappable-rc.workspace = true
futures.workspace = true
uuid.workspace = true

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,64 @@
[package]
name = "windmill-runtime-nativets"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_runtime_nativets"
path = "src/lib.rs"
[features]
default = []
private = []
enterprise = ["windmill-common/enterprise"]
[dependencies]
windmill-common = { workspace = true, default-features = false }
windmill-queue.workspace = true
windmill-parser-ts.workspace = true
deno_fetch.workspace = true
deno_webidl.workspace = true
deno_web.workspace = true
deno_net.workspace = true
deno_console.workspace = true
deno_url.workspace = true
deno_core.workspace = true
deno_ast.workspace = true
deno_tls.workspace = true
deno_permissions.workspace = true
deno_io.workspace = true
deno_telemetry.workspace = true
deno_error.workspace = true
deno_runtime.workspace = true
winapi.workspace = true
itertools.workspace = true
serde.workspace = true
serde_json.workspace = true
tokio.workspace = true
tracing.workspace = true
anyhow.workspace = true
uuid.workspace = true
reqwest.workspace = true
regex.workspace = true
lazy_static.workspace = true
const_format.workspace = true
futures.workspace = true
sqlx.workspace = true
[build-dependencies]
deno_fetch.workspace = true
deno_webidl.workspace = true
deno_web.workspace = true
deno_net.workspace = true
deno_console.workspace = true
deno_url.workspace = true
deno_core.workspace = true
deno_ast.workspace = true
deno_tls.workspace = true
deno_permissions.workspace = true
deno_io.workspace = true
deno_runtime.workspace = true
deno_telemetry.workspace = true
winapi.workspace = true

View File

@@ -0,0 +1,124 @@
use deno_fetch::FetchPermissions;
use deno_net::NetPermissions;
use deno_web::{BlobStore, TimersPermission};
use std::borrow::Cow;
use std::env;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct PermissionsContainer;
impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_net_url(
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
#[inline(always)]
fn check_read<'a>(
&mut self,
_resolved: bool,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
unreachable!("snapshotting")
}
}
impl TimersPermission for PermissionsContainer {
#[inline(always)]
fn allow_hrtime(&mut self) -> bool {
true
}
}
impl NetPermissions for PermissionsContainer {
fn check_read<'a>(
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write<'a>(
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_net<T: AsRef<str>>(
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write_path<'a>(
&mut self,
_: &'a Path,
_: &str,
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
todo!()
}
}
deno_core::extension!(
fetch,
esm_entry_point = "ext:fetch/src/runtime.js",
esm = ["src/runtime.js"],
);
fn main() {
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
Arc::new(BlobStore::default()),
None,
),
deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(None, None),
fetch::init_ops_and_esm(),
];
// Build the file path to the snapshot.
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
// Create the snapshot.
let output = deno_core::snapshot::create_snapshot(
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: Some(std::rc::Rc::new(|specifier, source| {
deno_runtime::transpile::maybe_transpile_source(specifier, source)
})),
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,
},
None,
)
.unwrap();
let mut file = std::fs::File::create(snapshot_path).unwrap();
file.write_all(&output.output).unwrap();
for path in output.files_loaded_during_snapshot {
println!("cargo:rerun-if-changed={}", path.display());
}
}

View File

@@ -0,0 +1,710 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Isolated deno_core runtime for NativeTS script execution.
//!
//! This crate encapsulates all deno_core/V8 dependencies for executing
//! TypeScript scripts via the nativets runtime. By isolating this here,
//! deno_core compilation no longer blocks windmill-worker or windmill-api.
use std::{borrow::Cow, cell::RefCell, path::PathBuf, rc::Rc, sync::Arc};
// Re-export deno_telemetry for use by windmill-worker's otel proxy
pub use deno_telemetry;
use deno_ast::ParseParams;
use deno_core::{
op2, serde_v8, url,
v8::{self, IsolateHandle},
Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
};
use deno_fetch::FetchPermissions;
use deno_net::NetPermissions;
use deno_web::{BlobStore, TimersPermission};
use itertools::Itertools;
use lazy_static::lazy_static;
use regex::Regex;
use serde_json::value::RawValue;
use sqlx::types::Json;
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
use windmill_common::error::Error;
use windmill_common::result_stream::append_result_stream_db;
use windmill_common::worker::{write_file, Connection, TMP_DIR};
// ── Permission container ─────────────────────────────────────────────
pub struct PermissionsContainer;
impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_net_url(
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
#[inline(always)]
fn check_read<'a>(
&mut self,
_resolved: bool,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
Ok(Cow::Borrowed(p))
}
}
impl TimersPermission for PermissionsContainer {
#[inline(always)]
fn allow_hrtime(&mut self) -> bool {
true
}
}
impl NetPermissions for PermissionsContainer {
fn check_read<'a>(
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
fn check_write<'a>(
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
fn check_net<T: AsRef<str>>(
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
fn check_write_path<'a>(
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<std::borrow::Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
// ── Types ────────────────────────────────────────────────────────────
struct MainArgs {
args: Vec<Option<Box<RawValue>>>,
}
struct LogString {
pub s: mpsc::UnboundedSender<String>,
}
pub struct NativeAnnotation {
pub useragent: Option<String>,
pub proxy: Option<(String, Option<(String, String)>)>,
}
// ── Statics ──────────────────────────────────────────────────────────
static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin"));
const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js");
const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors");
lazy_static! {
static ref RE_PROXY: Regex =
Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap();
}
// ── Public interface ─────────────────────────────────────────────────
/// Set up the deno_core/V8 runtime. Must be called once before creating any JsRuntime.
pub fn setup_deno_runtime() -> anyhow::Result<()> {
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
"--stack-size=1024".to_string(),
"--no-harmony-import-assertions".to_string(),
])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
if !unrecognized_v8_flags.is_empty() {
println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags);
}
deno_core::JsRuntime::init_platform(None, false);
Ok(())
}
pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
let parsed = deno_ast::parse_module(ParseParams {
specifier: url::Url::parse("file:///eval.ts")?,
capture_tokens: false,
scope_analysis: false,
media_type: deno_ast::MediaType::TypeScript,
maybe_syntax: None,
text: deno_core::ModuleCodeString::from(expr).into(),
})?;
Ok(parsed
.transpile(
&Default::default(),
&Default::default(),
&Default::default(),
)?
.into_source()
.text)
}
pub fn get_annotation(inner_content: &str) -> NativeAnnotation {
let mut res = NativeAnnotation { useragent: None, proxy: None };
let anns = inner_content
.lines()
.take_while(|x| x.starts_with("//"))
.map(|x| x.to_string().trim_start_matches("//").trim().to_string())
.collect_vec();
for ann in anns.iter() {
if ann.starts_with("useragent") {
res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string());
} else if ann.starts_with("proxy") {
res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim());
}
}
res
}
fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> {
RE_PROXY.captures(s).map(|x| {
(
format!(
"{}://{}{}",
x.get(1).map(|x| x.as_str()).unwrap_or_default(),
x.get(5).map(|x| x.as_str()).unwrap_or_default(),
x.get(7)
.map(|x| format!(":{}", x.as_str()))
.unwrap_or_default(),
),
x.get(3).map(|y| {
(
y.as_str().to_string(),
x.get(4).map(|x| x.as_str().to_string()).unwrap_or_default(),
)
}),
)
})
}
fn write_error_expr(expr: &str, uuid: &Uuid) {
if let Err(e) = std::fs::create_dir_all(ERROR_DIR) {
tracing::error!("failed to create error dir {ERROR_DIR}: {e}");
return;
}
let dir_entries = match std::fs::read_dir(ERROR_DIR) {
Ok(entries) => entries.count(),
Err(_) => {
tracing::error!("failed to read error dir {ERROR_DIR}");
return;
}
};
if std::env::var("PRINT_NATIVE_ERRORS").is_ok() {
tracing::info!("native error for job {uuid}: {expr}");
}
if dir_entries >= 100 {
tracing::info!("Too many error files in {ERROR_DIR}, skipping write");
return;
}
let path = format!("/{uuid}.js");
tracing::info!(
"nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}"
);
if let Err(e) = write_file(ERROR_DIR, &path, expr) {
tracing::error!("failed to write error expr to file {path}: {e}");
}
}
use windmill_common::utils::unsafe_raw;
async fn append_result_stream(
conn: &Connection,
workspace_id: &str,
job_id: &Uuid,
nstream: &str,
offset: i32,
) -> windmill_common::error::Result<()> {
match conn {
Connection::Sql(db) => {
append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?;
}
Connection::Http(client) => {
#[derive(serde::Serialize)]
struct ResultStreamBody<'a> {
result_stream: &'a str,
offset: i32,
}
let body = ResultStreamBody { result_stream: nstream, offset };
if let Err(e) = client
.post::<_, String>(
&format!(
"/api/w/{}/agent_workers/push_result_stream/{}",
workspace_id, job_id
),
None,
&body,
)
.await
{
tracing::error!(%job_id, %e, "error sending result stream for job {job_id}: {e}");
}
}
}
Ok(())
}
// ── ops ──────────────────────────────────────────────────────────────
#[op2]
#[serde]
fn op_get_static_args(op_state: Rc<RefCell<OpState>>) -> Vec<Option<String>> {
op_state
.borrow()
.borrow::<MainArgs>()
.args
.iter()
.map(|x| x.as_ref().map(|y| y.get().to_string()))
.collect_vec()
}
#[op2(fast)]
fn op_log(op_state: Rc<RefCell<OpState>>, #[string] log: &str) {
if let Err(e) = op_state
.borrow_mut()
.borrow_mut::<LogString>()
.s
.send(log.to_string())
{
tracing::error!("failed to send log: {e}");
}
}
// ── eval_fetch_timeout ───────────────────────────────────────────────
/// Execute a NativeTS script using deno_core/V8.
///
/// Returns `(result, has_stream)` where `has_stream` indicates if the result
/// came from an async iterable stream.
///
/// `otel_initialized` should be `DENO_OTEL_INITIALIZED.load(SeqCst)`.
///
/// The caller (windmill-worker) is responsible for wrapping this in
/// `run_future_with_polling_update_job_poller` for job cancellation/polling.
#[allow(clippy::too_many_arguments)]
pub async fn eval_fetch_timeout(
env_code: String,
ts_expr: String,
js_expr: String,
args: Option<&Json<std::collections::HashMap<String, Box<RawValue>>>>,
script_entrypoint_override: Option<String>,
job_id: Uuid,
conn: &Connection,
w_id: &str,
load_client: bool,
otel_initialized: bool,
stream_notifier_update: Option<Arc<dyn Fn() + Send + Sync + 'static>>,
) -> windmill_common::error::Result<(Box<RawValue>, bool)> {
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::<String>();
let (result_stream_sender, mut result_stream_receiver) = mpsc::unbounded_channel::<String>();
let conn_ = conn.clone();
let w_id_ = w_id.to_string();
tokio::spawn(async move {
while let Some(log) = append_logs_receiver.recv().await {
windmill_queue::append_logs(&job_id, &w_id_, log, &conn_).await
}
});
let append_result_stream_fn = append_result_stream;
let conn_ = conn.clone();
let w_id_ = w_id.to_string();
tokio::spawn(async move {
let mut offset = -1;
while let Some(stream) = result_stream_receiver.recv().await {
offset += 1;
if let Err(e) = append_result_stream_fn(&conn_, &w_id_, &job_id, &stream, offset).await
{
tracing::error!("failed to append result stream: {e}");
}
}
});
let parsed_args = windmill_parser_ts::parse_deno_signature(
&ts_expr,
true,
false,
script_entrypoint_override.clone(),
)?
.args;
let spread = parsed_args
.into_iter()
.map(|x| {
args.as_ref()
.and_then(|args| args.0.get(&x.name).map(|x| x.clone()))
})
.collect::<Vec<_>>();
let ann = get_annotation(&ts_expr);
#[cfg(not(feature = "enterprise"))]
if ann.proxy.is_some() {
return Err(Error::ExecutionErr("Proxy is an EE feature".to_string()).into());
}
let mut extra_logs = String::new();
if ann.useragent.is_some() {
extra_logs.push_str(&format!("useragent: {}\n", ann.useragent.as_ref().unwrap()));
}
if ann.proxy.is_some() {
let (proxy, auth) = ann.proxy.as_ref().unwrap();
extra_logs.push_str(&format!(
"proxy: {proxy} (basic auth: {})\n",
auth.is_some()
));
}
let result_f = tokio::task::spawn_blocking(move || {
let ops = vec![op_get_static_args(), op_log()];
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let fetch_options = deno_fetch::Options {
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
basic_auth: x
.1
.map(|(username, password)| deno_tls::BasicAuth { username, password }),
}),
..Default::default()
};
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::init_ops(),
deno_webidl::deno_webidl::init_ops(),
deno_url::deno_url::init_ops(),
deno_console::deno_console::init_ops(),
deno_web::deno_web::init_ops::<PermissionsContainer>(
Arc::new(BlobStore::default()),
None,
),
deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(fetch_options),
deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
ext,
];
let options = RuntimeOptions {
is_main: true,
extensions: exts,
create_params: Some(
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
),
startup_snapshot: Some(RUNTIME_SNAPSHOT),
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
extension_transpiler: None,
..Default::default()
};
let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>();
let mut js_runtime: JsRuntime = JsRuntime::new(options);
// Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized.
if otel_initialized {
if let Err(e) =
js_runtime.execute_script("<otel_bootstrap>", "globalThis.__bootstrapOtel()")
{
tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e);
}
}
js_runtime.add_near_heap_limit_callback(move |x, y| {
tracing::error!("heap limit reached: {x} {y}");
if memory_limit_tx.send(()).is_err() {
tracing::error!("failed to send memory limit reached notification - isolate may already be terminating");
};
y * 2
});
let (log_sender, mut log_receiver) = mpsc::unbounded_channel::<String>();
{
let op_state = js_runtime.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(PermissionsContainer {});
op_state.put(MainArgs { args: spread });
op_state.put(LogString { s: log_sender });
}
sender
.send(js_runtime.v8_isolate().thread_safe_handle())
.map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let future = async {
if !extra_logs.is_empty() {
if let Err(e) = append_logs_sender.send(extra_logs) {
tracing::error!("failed to send extra logs: {e}");
}
}
let handle = tokio::spawn(async move {
let mut result_stream = String::new();
let mut is_stream = false;
while let Some(log) = log_receiver.recv().await {
use windmill_common::result_stream::extract_stream_from_logs;
if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) {
if !is_stream {
is_stream = true;
if let Some(ref f) = stream_notifier_update {
f();
}
}
result_stream.push_str(&stream);
if let Err(e) = result_stream_sender.send(stream) {
tracing::error!("failed to send result stream: {e}");
}
} else {
if let Err(e) = append_logs_sender.send(log) {
tracing::error!("failed to send log: {e}");
}
}
}
if !result_stream.is_empty() {
Some(result_stream)
} else {
None
}
});
let r = tokio::select! {
r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id, otel_initialized) => Ok(r),
_ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string()))
};
drop(js_runtime);
if let Ok(r) = r {
match handle.await {
Ok(Some(logs)) => {
// merge_result_stream: if main result is null but stream exists, use stream
match r {
Ok(raw) if raw.get() == "null" => Ok((unsafe_raw(logs), true)),
Ok(raw) => Ok((raw, true)),
Err(e) => Err(e),
}
}
Ok(None) => Ok(r.map(|r| (r, false))?),
Err(e) => Err(Error::ExecutionErr(e.to_string())),
}
} else {
r.map(|r| r.map(|r| (r, false)))?
}
};
let r = runtime.block_on(future)?;
Ok(r) as windmill_common::error::Result<(Box<RawValue>, bool)>
});
let result = result_f.await.map_err(windmill_common::error::to_anyhow)?;
match result {
Ok((res, has_stream)) => Ok((res, has_stream)),
Err(e) => {
if let Ok(isolate) = receiver.try_recv() {
isolate.terminate_execution();
}
Err(e)
}
}
}
async fn eval_fetch(
js_runtime: &mut JsRuntime,
expr: &str,
env_code: Option<String>,
script_entrypoint_override: Option<String>,
load_client: bool,
job_id: &Uuid,
_otel_initialized: bool,
) -> windmill_common::error::Result<Box<RawValue>> {
if load_client {
if let Some(env_code) = env_code.as_ref() {
let _ = js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///windmill.ts")
.map_err(windmill_common::error::to_anyhow)?,
format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()),
)
.await
.map_err(windmill_common::error::to_anyhow)?;
}
}
use anyhow::Context;
use deno_core::error::CoreError;
use windmill_common::worker::to_raw_value;
let source = format!("{}\n{expr}", env_code.unwrap_or_default());
let _ = js_runtime
.load_side_es_module_from_code(
&deno_core::resolve_url("file:///eval.ts")
.map_err(windmill_common::error::to_anyhow)?,
source.to_string(),
)
.await
.map_err(|e| {
write_error_expr(expr, &job_id);
e
})
.context("failed to load module")?;
let main_override = script_entrypoint_override.unwrap_or("main".to_string());
#[cfg(all(feature = "private", feature = "enterprise"))]
let otel_context_inject = if _otel_initialized {
let trace_id = job_id.as_simple().to_string();
format!(
r#"globalThis.__enterSpan?.({{
isRecording: () => true,
spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }})
}});"#
)
} else {
String::new()
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
let otel_context_inject = "";
let script = js_runtime
.execute_script(
"<anon>",
format!(
r#"
function isAsyncIterable(obj) {{
return obj != null && typeof obj[Symbol.asyncIterator] === 'function';
}}
function processStreamIterative(res) {{
const iterator = res[Symbol.asyncIterator]();
function processLoop() {{
return new Promise(function(resolve) {{
function step() {{
iterator.next().then(function(result) {{
if (!result.done) {{
const chunk = result.value;
console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n'));
step();
}} else {{
resolve("null");
}}
}}).catch(function(error) {{
resolve("null");
}});
}}
step();
}});
}}
return processLoop();
}}
{otel_context_inject}
let args = Deno.core.ops.op_get_static_args().map(JSON.parse)
import("file:///eval.ts").then((module) => module.{main_override}(...args))
.then(res => {{
if (isAsyncIterable(res)) {{
return processStreamIterative(res)
}} else {{
return JSON.stringify(res ?? null);
}}
}})
"#
),
)
.map_err(|e| {
write_error_expr(expr, &job_id);
e
})
.context("native script initialization")?;
let fut = js_runtime.resolve(script);
let global = js_runtime
.with_event_loop_promise(fut, PollEventLoopOptions::default())
.await
.map_err(|e| {
write_error_expr(expr, &job_id);
e
});
match global {
Ok(global) => {
let scope = &mut js_runtime.handle_scope();
let local = v8::Local::new(scope, global);
let r = serde_v8::from_v8::<Option<String>>(scope, local)
.map_err(windmill_common::error::to_anyhow)?;
Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string())))
}
Err(CoreError::Js(e)) => {
let stack_head = e.frames.first().and_then(|f| {
if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") {
Some(format!(
"{}\n",
source
.lines()
.nth((f.line_number.unwrap_or(1)) as usize - 1)
.unwrap_or("")
.to_string()
))
} else {
None
}
});
let stack_s = format!(
"{}{}",
stack_head.unwrap_or("".to_string()),
e.stack.unwrap_or("".to_string())
);
let stack = if stack_s.is_empty() {
None
} else {
Some(stack_s)
};
Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({
"message": e.message,
"stack": stack,
"name": e.name,
}))))
}
Err(e) => Err(Error::ExecutionErr(e.print_with_cause())),
}
}

View File

@@ -0,0 +1,62 @@
import * as abortSignal from "ext:deno_web/03_abort_signal.js";
import * as base64 from "ext:deno_web/05_base64.js";
import * as console from "ext:deno_console/01_console.js";
import * as encoding from "ext:deno_web/08_text_encoding.js";
import * as event from "ext:deno_web/02_event.js";
import * as fetch from "ext:deno_fetch/26_fetch.js";
import * as file from "ext:deno_web/09_file.js";
import * as fileReader from "ext:deno_web/10_filereader.js";
import * as formData from "ext:deno_fetch/21_formdata.js";
import * as headers from "ext:deno_fetch/20_headers.js";
import * as streams from "ext:deno_web/06_streams.js";
import * as timers from "ext:deno_web/02_timers.js";
import * as url from "ext:deno_url/00_url.js";
import * as net from "ext:deno_net/01_net.js";
import * as tls from "ext:deno_net/02_tls.js";
import * as urlPattern from "ext:deno_url/01_urlpattern.js";
import * as webidl from "ext:deno_webidl/00_webidl.js";
import * as response from "ext:deno_fetch/23_response.js";
import * as request from "ext:deno_fetch/23_request.js";
import "ext:deno_web/02_structured_clone.js";
import "ext:deno_web/04_global_interfaces.js";
import "ext:deno_web/13_message_port.js";
import "ext:deno_web/14_compression.js";
import "ext:deno_web/15_performance.js";
import "ext:deno_web/16_image_data.js";
import "ext:deno_fetch/27_eventsource.js";
globalThis.atob = base64.atob;
globalThis.btoa = base64.btoa;
globalThis.fetch = fetch.fetch;
globalThis.Request = request.Request;
globalThis.Response = response.Response;
globalThis.Blob = file.Blob;
globalThis.URL = url.URL;
globalThis.FormData = formData.FormData;
globalThis.URLSearchParams = url.URLSearchParams;
globalThis.Headers = headers.Headers;
globalThis.FileReader = fileReader.FileReader;
globalThis.console = new console.Console((msg, level) =>
globalThis.Deno.core.ops.op_log(msg)
);
globalThis.AbortController = abortSignal.AbortController;
globalThis.AbortSignal = abortSignal.AbortSignal;
Object.assign(globalThis, {
clearInterval: timers.clearInterval,
clearTimeout: timers.clearTimeout,
setInterval: timers.setInterval,
setTimeout: timers.setTimeout,
});
// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
// consoleConfig: 0=ignore, 1=capture, 2=replace
globalThis.__bootstrapOtel = () => {
import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => {
bootstrap([1, 0, 1, 0]);
// Expose enterSpan for setting parent trace context
globalThis.__enterSpan = enterSpan;
});
};

View File

@@ -23,9 +23,7 @@ parquet = ["windmill-common/parquet", "dep:object_store"]
flow_testing = []
cloud = []
sqlx = []
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile",
"quickjs"]
deno_core = ["dep:windmill-runtime-nativets"]
libffi_mac = ["dep:libffi-sys"]
otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"]
dind = ["dep:bollard"]
@@ -39,7 +37,6 @@ nu = ["dep:windmill-parser-nu"]
java = ["dep:windmill-parser-java"]
ruby = ["dep:windmill-parser-ruby"]
duckdb = ["dep:libloading"]
quickjs = ["dep:rquickjs"]
bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"]
[dependencies]
@@ -47,6 +44,8 @@ windmill-queue.workspace = true
windmill-dep-map.workspace = true
windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker.
windmill-common = { workspace = true, default-features = false }
windmill-jseval.workspace = true
windmill-runtime-nativets = { workspace = true, optional = true }
windmill-mcp = { workspace = true, optional = true }
windmill-macros.workspace = true
windmill-parser.workspace = true
@@ -97,20 +96,6 @@ once_cell.workspace = true
tokio-postgres.workspace = true
bit-vec.workspace = true
url.workspace = true
deno_telemetry = { workspace = true, optional = true }
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
deno_web = { workspace = true, optional = true }
deno_net = { workspace = true, optional = true }
deno_console = { workspace = true, optional = true }
deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
rustls-pemfile = { workspace = true, optional = true }
async-stream.workspace = true
postgres-native-tls.workspace = true
@@ -137,7 +122,6 @@ object_store = { workspace = true, optional = true}
convert_case.workspace = true
yaml-rust.workspace = true
backon.workspace = true
winapi = { workspace = true, optional = true }
pep440_rs.workspace = true
process-wrap.workspace = true
async-once-cell.workspace = true
@@ -150,7 +134,6 @@ prost.workspace = true
axum.workspace = true
bollard = { workspace = true, optional = true }
oracle = { workspace = true, optional = true }
rquickjs = { workspace = true, optional = true }
hudsucker.workspace = true
hyper-http-proxy.workspace = true
hyper-tls.workspace = true
@@ -158,18 +141,4 @@ hyper-util.workspace = true
rcgen.workspace = true
[build-dependencies]
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
deno_web = { workspace = true, optional = true }
deno_net = { workspace = true, optional = true }
deno_console = { workspace = true, optional = true }
deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_runtime = { workspace = true, optional = true }
deno_telemetry = { workspace = true, optional = true }
winapi = { workspace = true, optional = true }
libffi-sys = { workspace = true, optional = true }

View File

@@ -1,154 +1 @@
#[cfg(feature = "deno_core")]
use deno_fetch::FetchPermissions;
#[cfg(feature = "deno_core")]
use deno_net::NetPermissions;
#[cfg(feature = "deno_core")]
use deno_web::{BlobStore, TimersPermission};
#[cfg(feature = "deno_core")]
use std::borrow::Cow;
#[cfg(feature = "deno_core")]
use std::env;
#[cfg(feature = "deno_core")]
use std::io::Write;
#[cfg(feature = "deno_core")]
use std::path::{Path, PathBuf};
#[cfg(feature = "deno_core")]
use std::sync::Arc;
// #[cfg(feature = "deno_core")]
pub struct PermissionsContainer;
#[cfg(feature = "deno_core")]
impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_net_url(
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
#[inline(always)]
fn check_read<'a>(
&mut self,
_resolved: bool,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
unreachable!("snapshotting")
}
}
#[cfg(feature = "deno_core")]
impl TimersPermission for PermissionsContainer {
#[inline(always)]
fn allow_hrtime(&mut self) -> bool {
true
}
}
#[cfg(feature = "deno_core")]
impl NetPermissions for PermissionsContainer {
fn check_read<'a>(
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write<'a>(
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_net<T: AsRef<str>>(
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write_path<'a>(
&mut self,
_: &'a Path,
_: &str,
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
todo!()
}
}
#[cfg(feature = "deno_core")]
deno_core::extension!(
fetch,
esm_entry_point = "ext:fetch/src/runtime.js",
esm = ["src/runtime.js"],
);
#[cfg(feature = "deno_core")]
fn main() {
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
Arc::new(BlobStore::default()),
None,
),
deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(None, None),
fetch::init_ops_and_esm(),
];
// Build the file path to the snapshot.
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
// Create the snapshot.
let output = deno_core::snapshot::create_snapshot(
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: Some(std::rc::Rc::new(|specifier, source| {
deno_runtime::transpile::maybe_transpile_source(specifier, source)
})),
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,
},
None,
)
.unwrap();
// NOTE(bartlomieju): Compressing the TSC snapshot in debug build took
// ~45s on M1 MacBook Pro; without compression it took ~1s.
// Thus we're not using compressed snapshot, trading off
// a lot of build time for some startup time in debug build.
let mut file = std::fs::File::create(snapshot_path).unwrap();
// if cfg!(debug_assertions) {
file.write_all(&output.output).unwrap();
// } else {
// let mut vec = Vec::with_capacity(output.output.len());
// vec.extend((output.output.len() as u32).to_le_bytes());
// vec.extend_from_slice(
// &zstd::bulk::compress(&output.output, 22).expect("snapshot compression failed"),
// );
// file.write_all(&vec).unwrap();
// }
for path in output.files_loaded_during_snapshot {
println!("cargo:rerun-if-changed={}", path.display());
}
}
#[cfg(not(feature = "deno_core"))]
fn main() {}

View File

@@ -333,10 +333,7 @@ pub async fn read_file_bytes(path: &str) -> error::Result<Vec<u8>> {
Ok(content)
}
//this skips more steps than from_str at the cost of being unsafe. The source must ALWAUS gemerate valid json or this can cause UB in the worst case
pub fn unsafe_raw(json: String) -> Box<RawValue> {
unsafe { std::mem::transmute::<Box<str>, Box<RawValue>>(json.into()) }
}
pub use windmill_common::utils::unsafe_raw;
fn check_result_too_big(size: usize) -> error::Result<()> {
if *CLOUD_HOSTED && size > MAX_RESULT_SIZE {

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,957 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! QuickJS-based JavaScript expression evaluation for flow transformations.
//!
//! This module provides an alternative to deno_core for evaluating arbitrary JavaScript
//! expressions in flow transformations. QuickJS offers significantly faster startup times
//! (~200μs vs ~3ms for V8), making it ideal for evaluating many small expressions.
//!
//! ## Performance Characteristics (release mode benchmarks)
//! - **Simple expressions**: ~238μs (QuickJS) vs ~3.05ms (deno_core) = **~13x faster**
//! - **Complex expressions**: ~192μs (QuickJS) vs ~3.09ms (deno_core) = **~16x faster**
//! - **Memory**: ~2.5% of V8's footprint
//!
//! For flow expression evaluation where startup time dominates, QuickJS is significantly
//! faster overall despite being slower for long-running CPU-intensive code.
//!
//! ## Async Operations
//! This implementation uses true async Rust callbacks (similar to deno_core's ops) for
//! `variable()`, `resource()`, and `results.xxx` access. The async functions use
//! rquickjs's `Async<MutFn<...>>` wrapper which returns JavaScript Promises that are
//! resolved when the Rust async operations complete. No pre-fetching is required.
use std::collections::HashMap;
use std::sync::Arc;
use rquickjs::{
async_with,
prelude::{Async, Func, MutFn},
AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object, Value,
};
use serde_json::value::RawValue;
use windmill_common::client::AuthedClient;
use windmill_common::flow_status::JobResult;
use crate::js_eval::{replace_with_await, replace_with_await_result, IdContext};
/// Shared state for async operations within QuickJS
#[derive(Clone)]
struct AsyncOpState {
client: AuthedClient,
}
/// Evaluates a JavaScript expression using QuickJS runtime.
///
/// This function provides the same interface as `eval_timeout` but uses QuickJS
/// instead of deno_core/V8 for potentially faster startup times.
///
/// Unlike deno_core, this uses true async Rust callbacks for `variable()`,
/// `resource()`, and `results.xxx` access - no pre-fetching required.
pub async fn eval_timeout_quickjs(
expr: String,
transform_context: HashMap<String, Arc<Box<RawValue>>>,
flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
flow_env: Option<&HashMap<String, Box<RawValue>>>,
authed_client: Option<&AuthedClient>,
by_id: Option<&IdContext>,
ctx: Option<Vec<(String, String)>>,
) -> anyhow::Result<Box<RawValue>> {
let expr = expr.trim().to_string();
tracing::debug!(
"evaluating js eval (quickjs): {} with context {:?}",
expr,
transform_context
);
// Clone data for the blocking task
let by_id_clone = by_id.cloned();
let flow_input_clone = flow_input.clone();
let flow_env_clone = flow_env.cloned();
let authed_client_clone = authed_client.cloned();
// Determine which context keys are actually used in the expression
let p_ids = by_id.map(|x| {
[
format!("results.{}", x.previous_id),
format!("results?.{}", x.previous_id),
format!("results[\"{}\"]", x.previous_id),
format!("results?.[\"{}\"]", x.previous_id),
]
});
let mut context_keys: Vec<String> = transform_context
.keys()
.filter(|x| expr.contains(&x.to_string()))
.cloned()
.collect();
if !context_keys.contains(&"previous_result".to_string())
&& (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x)))
|| expr.contains("error")
{
context_keys.push("previous_result".to_string());
}
let has_flow_input = expr.contains("flow_input");
if has_flow_input {
context_keys.push("flow_input".to_string())
}
// Filter transform_context to only include used keys
let filtered_context: HashMap<String, Arc<Box<RawValue>>> = transform_context
.into_iter()
.filter(|(k, _)| context_keys.contains(k))
.collect();
let expr_clone = expr.clone();
// Run the QuickJS evaluation with a timeout
tokio::time::timeout(
std::time::Duration::from_millis(10000),
tokio::task::spawn_blocking(move || {
// Create a new tokio runtime for async operations within the blocking context
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
rt.block_on(async move {
eval_quickjs_inner(
&expr_clone,
filtered_context,
flow_input_clone,
flow_env_clone,
authed_client_clone,
by_id_clone,
ctx,
context_keys,
)
.await
})
}),
)
.await
.map_err(|_| {
anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)")
})??
}
/// Memory limit for QuickJS runtime (32MB).
/// This is much smaller than deno_core's 128MB limit since flow expressions
/// should be lightweight transformations, not memory-intensive operations.
const QUICKJS_MEMORY_LIMIT: usize = 32 * 1024 * 1024;
async fn eval_quickjs_inner(
expr: &str,
transform_context: HashMap<String, Arc<Box<RawValue>>>,
flow_input: Option<mappable_rc::Marc<HashMap<String, Box<RawValue>>>>,
flow_env: Option<HashMap<String, Box<RawValue>>>,
authed_client: Option<AuthedClient>,
by_id: Option<IdContext>,
extra_ctx: Option<Vec<(String, String)>>,
context_keys: Vec<String>,
) -> anyhow::Result<Box<RawValue>> {
let runtime = AsyncRuntime::new()?;
runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await;
let context = AsyncContext::full(&runtime).await?;
// Create shared state for async ops if we have a client
let op_state = authed_client.map(|client| Arc::new(AsyncOpState { client }));
let op_state_clone = op_state.clone();
let by_id_clone = by_id.clone();
// Transform expression to add await for variable/resource/results access
let expr_with_funcs = ["variable", "resource"]
.into_iter()
.fold(expr.to_string(), replace_with_await);
let transformed_expr = replace_with_await_result(expr_with_funcs);
async_with!(context => |ctx| {
let globals = ctx.globals();
// Set up context variables
for key in &context_keys {
if key == "flow_input" {
if let Some(ref fi) = flow_input {
let json_str = serde_json::to_string(fi.as_ref())?;
let val: serde_json::Value = serde_json::from_str(&json_str)?;
let js_val = json_to_js(&ctx, &val)?;
globals.set(key.as_str(), js_val)?;
} else {
globals.set(key.as_str(), Value::new_null(ctx.clone()))?;
}
} else if let Some(raw_val) = transform_context.get(key) {
let val: serde_json::Value = serde_json::from_str(raw_val.get())?;
let js_val = json_to_js(&ctx, &val)?;
globals.set(key.as_str(), js_val)?;
}
}
// Set up flow_env if referenced
if expr.contains("flow_env") {
if let Some(ref fe) = flow_env {
let obj = Object::new(ctx.clone())?;
for (k, v) in fe {
let val: serde_json::Value = serde_json::from_str(v.get())?;
let js_val = json_to_js(&ctx, &val)?;
obj.set(k.as_str(), js_val)?;
}
globals.set("flow_env", obj)?;
} else {
globals.set("flow_env", Object::new(ctx.clone())?)?;
}
}
// Set up additional context variables
if let Some(ctx_vars) = extra_ctx {
for (k, v) in ctx_vars {
globals.set(k.as_str(), v.as_str())?;
}
}
// Set up error extraction if needed
if expr.contains("error") && context_keys.contains(&"previous_result".to_string()) {
let error_setup = r#"
let error = previous_result?.error;
if (!error) {
if (Array.isArray(previous_result)) {
const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item);
if (errors.length === 1) {
error = errors[0].error;
} else if (errors.length > 1) {
error = {
name: 'MultipleErrors',
message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '),
errors: previous_result
};
} else {
error = {
name: 'MultipleErrors',
message: "Could not parse errors",
errors: previous_result
};
}
} else {
if (previous_result) {
error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result };
} else {
error = { name: 'UnknownError', message: 'No error found' };
}
}
}
"#;
ctx.eval::<(), _>(error_setup).catch(&ctx).map_err(quickjs_error_to_anyhow)?;
}
// Set up async functions if we have a client
if let Some(ref state) = op_state_clone {
setup_async_ops(&ctx, &globals, state.clone())?;
} else {
// Set up stub functions that throw errors
setup_stub_functions(&ctx, &globals)?;
}
// Set up results proxy if we have by_id context
if let Some(ref by_id) = by_id_clone {
setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?;
}
// Determine if we need to add return statement.
// Wrap with .then((x) => JSON.stringify(x ?? null)) to serialize the result
// using the standard JSON.stringify, matching deno_core's behavior exactly.
let code = if should_add_return_quickjs(&transformed_expr) {
format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr)
} else {
format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr)
};
// Evaluate the expression (returns a Promise that resolves to a JSON string)
let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?;
// Await the promise — result is already a JSON string from JSON.stringify
let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?;
let json_str = String::from_js(&ctx, result)
.unwrap_or_else(|_| "null".to_string());
Ok(crate::common::unsafe_raw(json_str))
})
.await
}
/// Set up async variable() and resource() functions using true Rust async callbacks.
///
/// This uses rquickjs's `Async<MutFn<...>>` wrapper to create JavaScript functions that
/// return Promises. The Promises are resolved by spawned Rust async operations.
fn setup_async_ops<'js>(
ctx: &rquickjs::Ctx<'js>,
globals: &Object<'js>,
state: Arc<AsyncOpState>,
) -> anyhow::Result<()> {
// Error prefix - must match the JavaScript side
const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00";
// Create variable() function with true async Rust callback
// Returns a JSON string that JavaScript will parse
let state_for_var = state.clone();
globals.set(
"__fetchVariable",
Func::from(Async(MutFn::new(move |path: String| {
let client = state_for_var.client.clone();
async move {
match client.get_variable_value(&path).await {
Ok(value) => value,
Err(e) => format!("{}{}", ERR_PREFIX, e),
}
}
}))),
)?;
// Create resource() function - returns JSON string
let state_for_res = state.clone();
globals.set(
"__fetchResource",
Func::from(Async(MutFn::new(move |path: String| {
let client = state_for_res.client.clone();
async move {
match client
.get_resource_value_interpolated::<serde_json::Value>(&path, None)
.await
{
Ok(value) => {
serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string())
}
Err(e) => format!("{}{}", ERR_PREFIX, e),
}
}
}))),
)?;
// Create JavaScript wrappers that parse the JSON results
// We use a unique prefix that's extremely unlikely to appear in real data
let wrapper_code = r#"
const __ERR_PREFIX = '\x00__WINDMILL_ERR__\x00';
async function variable(path) {
const result = await __fetchVariable(path);
if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) {
throw new Error(result.substring(__ERR_PREFIX.length));
}
return result;
}
async function resource(path) {
const result = await __fetchResource(path);
if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) {
throw new Error(result.substring(__ERR_PREFIX.length));
}
return JSON.parse(result);
}
"#;
ctx.eval::<(), _>(wrapper_code)
.catch(ctx)
.map_err(quickjs_error_to_anyhow)?;
Ok(())
}
/// Set up stub functions that throw errors when no client is available
fn setup_stub_functions<'js>(
ctx: &rquickjs::Ctx<'js>,
_globals: &Object<'js>,
) -> anyhow::Result<()> {
let setup_code = r#"
function variable(path) {
return Promise.reject(new Error(`variable() is not available without an authenticated client`));
}
function resource(path) {
return Promise.reject(new Error(`resource() is not available without an authenticated client`));
}
"#;
ctx.eval::<(), _>(setup_code)
.catch(ctx)
.map_err(quickjs_error_to_anyhow)?;
Ok(())
}
/// Set up the `results` Proxy object with dynamic access to step results.
///
/// Uses async Rust callbacks to fetch results on-demand when accessed.
fn setup_results_proxy<'js>(
ctx: &rquickjs::Ctx<'js>,
globals: &Object<'js>,
by_id: &IdContext,
op_state: Option<Arc<AsyncOpState>>,
) -> anyhow::Result<()> {
// Store previous_id for the shortcut optimization
globals.set("__previous_id", by_id.previous_id.clone())?;
// Create async __getResult function that fetches step results via Rust
if let Some(state) = op_state {
let by_id_for_result = by_id.clone();
globals.set(
"__fetchResult",
Func::from(Async(MutFn::new(move |step_id: String| {
let client = state.client.clone();
let by_id = by_id_for_result.clone();
let step_id_clone = step_id.clone();
// Look up the job ID(s) for this step from the local cache
let job_result = by_id.steps_results.get(&step_id).cloned();
let flow_job_id = by_id.flow_job.to_string();
async move {
const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00";
let result: Result<serde_json::Value, String> = match job_result {
Some(jr) => {
// Found in local cache, fetch result by job ID
match jr {
JobResult::SingleJob(job_id) => client
.get_completed_job_result::<serde_json::Value>(
&job_id.to_string(),
None,
)
.await
.map_err(|e| {
format!(
"Failed to fetch result for step '{}': {}",
step_id_clone, e
)
}),
JobResult::ListJob(job_ids) => {
let futs = job_ids.iter().map(|job_id| {
let client = client.clone();
let job_id_str = job_id.to_string();
async move {
client
.get_completed_job_result::<serde_json::Value>(
&job_id_str,
None,
)
.await
}
});
let results: Vec<_> = futures::future::join_all(futs).await;
let collected: Result<Vec<_>, _> =
results.into_iter().collect();
collected.map(serde_json::Value::Array).map_err(|e| {
format!(
"Failed to fetch results for step '{}': {}",
step_id_clone, e
)
})
}
}
}
None => {
// Not in local cache, fallback to querying by flow_job_id and step_id
// This happens for branch modules that need to access parent flow step results
// Use .ok() to match deno_core behavior: return null for non-existent steps
// instead of throwing an error
Ok(client
.get_result_by_id::<serde_json::Value>(
&flow_job_id,
&step_id_clone,
None,
)
.await
.ok() // Swallow errors, convert to Option
.unwrap_or(serde_json::Value::Null)) // None -> null
}
};
match result {
Ok(value) => {
serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string())
}
Err(e) => format!("{}{}", ERR_PREFIX, e),
}
}
}))),
)?;
// Create JavaScript wrapper that parses the JSON result
let wrapper_code = r#"
const __RESULT_ERR_PREFIX = '\x00__WINDMILL_ERR__\x00';
async function __getResult(stepId) {
const result = await __fetchResult(stepId);
if (typeof result === 'string' && result.startsWith(__RESULT_ERR_PREFIX)) {
throw new Error(result.substring(__RESULT_ERR_PREFIX.length));
}
return JSON.parse(result);
}
"#;
ctx.eval::<(), _>(wrapper_code)
.catch(ctx)
.map_err(quickjs_error_to_anyhow)?;
} else {
// No client - stub function that rejects
let stub_code = r#"
function __getResult(stepId) {
return Promise.reject(new Error('Result fetching not available without authenticated client'));
}
"#;
ctx.eval::<(), _>(stub_code)
.catch(ctx)
.map_err(quickjs_error_to_anyhow)?;
}
// Create the results proxy that calls __getResult for on-demand fetching
// Matches deno_core behavior: always try to fetch, let backend handle unknown step IDs
let proxy_setup = r#"
const results = new Proxy({}, {
get: function(target, name, receiver) {
// Handle symbol properties (like Symbol.toStringTag)
if (typeof name === 'symbol') {
return undefined;
}
// Check if it's the previous_id and previous_result exists
if (name === __previous_id && typeof previous_result !== 'undefined') {
return Promise.resolve(previous_result);
}
// Always try to fetch - let Rust/backend handle unknown step IDs
// This matches deno_core behavior
return __getResult(name);
}
});
"#;
ctx.eval::<(), _>(proxy_setup)
.catch(ctx)
.map_err(quickjs_error_to_anyhow)?;
Ok(())
}
/// Convert a serde_json::Value to a QuickJS Value
fn json_to_js<'js>(
ctx: &rquickjs::Ctx<'js>,
val: &serde_json::Value,
) -> rquickjs::Result<Value<'js>> {
match val {
serde_json::Value::Null => Ok(Value::new_null(ctx.clone())),
serde_json::Value::Bool(b) => Ok(Value::new_bool(ctx.clone(), *b)),
serde_json::Value::Number(n) => {
if let Some(i) = n.as_i64() {
if i >= i32::MIN as i64 && i <= i32::MAX as i64 {
Ok(Value::new_int(ctx.clone(), i as i32))
} else {
Ok(Value::new_float(ctx.clone(), i as f64))
}
} else if let Some(f) = n.as_f64() {
Ok(Value::new_float(ctx.clone(), f))
} else {
Ok(Value::new_float(ctx.clone(), 0.0))
}
}
serde_json::Value::String(s) => s.clone().into_js(ctx),
serde_json::Value::Array(arr) => {
let js_arr = rquickjs::Array::new(ctx.clone())?;
for (i, item) in arr.iter().enumerate() {
js_arr.set(i, json_to_js(ctx, item)?)?;
}
Ok(js_arr.into_value())
}
serde_json::Value::Object(obj) => {
let js_obj = Object::new(ctx.clone())?;
for (k, v) in obj {
js_obj.set(k.as_str(), json_to_js(ctx, v)?)?;
}
Ok(js_obj.into_value())
}
}
}
/// Convert a QuickJS Value to a serde_json::Value
///
/// This mimics JavaScript's JSON.stringify behavior:
/// - For objects with a `toJSON` method (like Date), call it and use the result
/// - Arrays are recursively serialized
/// - Plain objects enumerate their own properties
/// Determines if we should prepend "return" to the expression
fn should_add_return_quickjs(expr: &str) -> bool {
let trimmed = expr.trim();
if trimmed.is_empty() {
return true;
}
if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" {
return false;
}
let statement_prefixes = [
"const ",
"let ",
"var ",
"if ",
"if(",
"for ",
"for(",
"while ",
"while(",
"switch ",
"switch(",
"try ",
"try{",
"throw ",
"function ",
"class ",
"async ",
"await ",
];
for prefix in &statement_prefixes {
if trimmed.starts_with(prefix) {
return false;
}
}
if contains_semicolon_outside_strings(trimmed) {
return false;
}
true
}
fn contains_semicolon_outside_strings(expr: &str) -> bool {
let mut in_single_quote = false;
let mut in_double_quote = false;
let mut in_template = false;
let mut prev_char = '\0';
for ch in expr.chars() {
match ch {
'\'' if prev_char != '\\' && !in_double_quote && !in_template => {
in_single_quote = !in_single_quote;
}
'"' if prev_char != '\\' && !in_single_quote && !in_template => {
in_double_quote = !in_double_quote;
}
'`' if prev_char != '\\' && !in_single_quote && !in_double_quote => {
in_template = !in_template;
}
';' if !in_single_quote && !in_double_quote && !in_template => {
return true;
}
_ => {}
}
prev_char = ch;
}
false
}
fn quickjs_error_to_anyhow(err: rquickjs::CaughtError<'_>) -> anyhow::Error {
anyhow::anyhow!("QuickJS evaluation error: {}", err)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use windmill_common::worker::to_raw_value;
#[tokio::test]
async fn test_eval_quickjs_simple() -> anyhow::Result<()> {
let mut env = HashMap::new();
env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5))));
env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3))));
let result =
eval_timeout_quickjs("x + y".to_string(), env, None, None, None, None, None).await?;
assert_eq!(result.get(), "8");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_object_access() -> anyhow::Result<()> {
let mut env = HashMap::new();
env.insert(
"params".to_string(),
Arc::new(to_raw_value(&json!({"test": 42, "nested": {"value": 100}}))),
);
let result = eval_timeout_quickjs(
"params.test".to_string(),
env.clone(),
None,
None,
None,
None,
None,
)
.await?;
assert_eq!(result.get(), "42");
let result2 = eval_timeout_quickjs(
"params.nested.value".to_string(),
env,
None,
None,
None,
None,
None,
)
.await?;
assert_eq!(result2.get(), "100");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_array() -> anyhow::Result<()> {
let mut env = HashMap::new();
env.insert(
"arr".to_string(),
Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))),
);
let result = eval_timeout_quickjs(
"arr.map(x => x * 2)".to_string(),
env,
None,
None,
None,
None,
None,
)
.await?;
assert_eq!(result.get(), "[2,4,6,8,10]");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_flow_input() -> anyhow::Result<()> {
let mut flow_input = HashMap::new();
flow_input.insert("name".to_string(), to_raw_value(&json!("test")));
flow_input.insert("count".to_string(), to_raw_value(&json!(10)));
let result = eval_timeout_quickjs(
"flow_input.name".to_string(),
HashMap::new(),
Some(mappable_rc::Marc::new(flow_input)),
None,
None,
None,
None,
)
.await?;
assert_eq!(result.get(), "\"test\"");
Ok(())
}
#[test]
fn test_should_add_return_quickjs() {
assert!(should_add_return_quickjs("5"));
assert!(should_add_return_quickjs("x + y"));
assert!(should_add_return_quickjs("foo()"));
assert!(!should_add_return_quickjs("return 5"));
assert!(!should_add_return_quickjs("return x + y"));
assert!(!should_add_return_quickjs("const x = 5"));
assert!(!should_add_return_quickjs("let y = 10"));
assert!(!should_add_return_quickjs("if (x > 5) { return x; }"));
assert!(!should_add_return_quickjs("let x = 5; x + 1"));
}
#[tokio::test]
async fn test_eval_quickjs_date_serialization() -> anyhow::Result<()> {
// Test that Date objects serialize to ISO strings, matching JSON.stringify behavior
let result = eval_timeout_quickjs(
"new Date('2024-01-15T12:30:00.000Z')".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Should be an ISO string, not an empty object
assert_eq!(result.get(), "\"2024-01-15T12:30:00.000Z\"");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_date_in_object() -> anyhow::Result<()> {
// Test that Date objects within other objects serialize correctly
let result = eval_timeout_quickjs(
"({ date: new Date('2024-01-15T12:30:00.000Z'), name: 'test' })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value["date"], "2024-01-15T12:30:00.000Z");
assert_eq!(value["name"], "test");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_date_in_array() -> anyhow::Result<()> {
// Test that Date objects in arrays serialize correctly
let result = eval_timeout_quickjs(
"[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]"
.to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value[0], "2024-01-15T00:00:00.000Z");
assert_eq!(value[1], "2024-01-16T00:00:00.000Z");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_custom_tojson() -> anyhow::Result<()> {
// Test that JSON.stringify handles custom toJSON when returning objects
let result = eval_timeout_quickjs(
r#"({ a: 1, toJSON: () => ({ converted: true }) })"#.to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// JSON.stringify should call toJSON and use that result
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(value["converted"], true);
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_deeply_nested_date() -> anyhow::Result<()> {
// Test that Date objects deep in nested structures are handled
let result = eval_timeout_quickjs(
"({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
let value: serde_json::Value = serde_json::from_str(result.get())?;
assert_eq!(
value["level1"]["level2"]["date"],
"2024-01-15T00:00:00.000Z"
);
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_regexp_serialization() -> anyhow::Result<()> {
// RegExp objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"/test/gi".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// RegExp doesn't have toJSON, so it serializes to an empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_map_serialization() -> anyhow::Result<()> {
// Map objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"new Map([['key', 'value']])".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Map doesn't have toJSON, serializes to empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_set_serialization() -> anyhow::Result<()> {
// Set objects serialize to empty objects in JSON (same as JSON.stringify behavior)
let result = eval_timeout_quickjs(
"new Set([1, 2, 3])".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
// Set doesn't have toJSON, serializes to empty object (same as Deno)
assert_eq!(result.get(), "{}");
Ok(())
}
#[tokio::test]
async fn test_eval_quickjs_tojson_returning_date() -> anyhow::Result<()> {
// When toJSON returns a Date object, JSON.stringify does NOT call toJSON again
// on the returned value (per spec). Date has no own enumerable properties,
// so it serializes to {}.
let result = eval_timeout_quickjs(
"({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })".to_string(),
HashMap::new(),
None,
None,
None,
None,
None,
)
.await?;
assert_eq!(result.get(), "{}");
Ok(())
}
}

View File

@@ -39,10 +39,6 @@ pub mod job_logger;
pub mod job_logger_ee;
mod job_logger_oss;
mod js_eval;
#[cfg(test)]
mod js_eval_parity_tests;
#[cfg(feature = "quickjs")]
pub mod js_eval_quickjs;
pub mod memory_common;
#[cfg(feature = "private")]
pub mod memory_ee;

View File

@@ -1,212 +0,0 @@
import * as abortSignal from "ext:deno_web/03_abort_signal.js";
import * as base64 from "ext:deno_web/05_base64.js";
import * as console from "ext:deno_console/01_console.js";
import * as encoding from "ext:deno_web/08_text_encoding.js";
import * as event from "ext:deno_web/02_event.js";
import * as fetch from "ext:deno_fetch/26_fetch.js";
import * as file from "ext:deno_web/09_file.js";
import * as fileReader from "ext:deno_web/10_filereader.js";
import * as formData from "ext:deno_fetch/21_formdata.js";
import * as headers from "ext:deno_fetch/20_headers.js";
import * as streams from "ext:deno_web/06_streams.js";
import * as timers from "ext:deno_web/02_timers.js";
import * as url from "ext:deno_url/00_url.js";
import * as net from "ext:deno_net/01_net.js";
import * as tls from "ext:deno_net/02_tls.js";
import * as urlPattern from "ext:deno_url/01_urlpattern.js";
import * as webidl from "ext:deno_webidl/00_webidl.js";
import * as response from "ext:deno_fetch/23_response.js";
import * as request from "ext:deno_fetch/23_request.js";
import "ext:deno_web/02_structured_clone.js";
import "ext:deno_web/04_global_interfaces.js";
import "ext:deno_web/13_message_port.js";
import "ext:deno_web/14_compression.js";
import "ext:deno_web/15_performance.js";
import "ext:deno_web/16_image_data.js";
import "ext:deno_fetch/27_eventsource.js";
globalThis.atob = base64.atob;
globalThis.btoa = base64.btoa;
globalThis.fetch = fetch.fetch;
globalThis.Request = request.Request;
globalThis.Response = response.Response;
globalThis.Blob = file.Blob;
globalThis.URL = url.URL;
globalThis.FormData = formData.FormData;
globalThis.URLSearchParams = url.URLSearchParams;
globalThis.Headers = headers.Headers;
globalThis.FileReader = fileReader.FileReader;
globalThis.console = new console.Console((msg, level) =>
globalThis.Deno.core.ops.op_log(msg)
);
globalThis.AbortController = abortSignal.AbortController;
globalThis.AbortSignal = abortSignal.AbortSignal;
Object.assign(globalThis, {
clearInterval: timers.clearInterval,
clearTimeout: timers.clearTimeout,
setInterval: timers.setInterval,
setTimeout: timers.setTimeout,
});
// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
// consoleConfig: 0=ignore, 1=capture, 2=replace
globalThis.__bootstrapOtel = () => {
import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => {
bootstrap([1, 0, 1, 0]);
// Expose enterSpan for setting parent trace context
globalThis.__enterSpan = enterSpan;
});
};
// Object.assign(globalThis, {
// console: nonEnumerable(
// new console.Console((msg, level) => core.print(msg, level > 1))
// ),
// // timers
// // fetch
// Request: nonEnumerable(request.Request),
// Response: nonEnumerable(response.Response),
// Headers: nonEnumerable(headers.Headers),
// fetch: writable(fetch.fetch),
// // base64
// atob: writable(base64.atob),
// btoa: writable(base64.btoa),
// // encoding
// TextDecoder: nonEnumerable(encoding.TextDecoder),
// TextEncoder: nonEnumerable(encoding.TextEncoder),
// TextDecoderStream: nonEnumerable(encoding.TextDecoderStream),
// TextEncoderStream: nonEnumerable(encoding.TextEncoderStream),
// // url
// URL: nonEnumerable(url.URL),
// URLPattern: nonEnumerable(urlPattern.URLPattern),
// URLSearchParams: nonEnumerable(url.URLSearchParams),
// // // crypto
// // CryptoKey: nonEnumerable(crypto.CryptoKey),
// // crypto: readOnly(crypto.crypto),
// // Crypto: nonEnumerable(crypto.Crypto),
// // SubtleCrypto: nonEnumerable(crypto.SubtleCrypto),
// // streams
// ByteLengthQueuingStrategy: nonEnumerable(streams.ByteLengthQueuingStrategy),
// CountQueuingStrategy: nonEnumerable(streams.CountQueuingStrategy),
// ReadableStream: nonEnumerable(streams.ReadableStream),
// ReadableStreamDefaultReader: nonEnumerable(
// streams.ReadableStreamDefaultReader
// ),
// ReadableByteStreamController: nonEnumerable(
// streams.ReadableByteStreamController
// ),
// ReadableStreamBYOBReader: nonEnumerable(streams.ReadableStreamBYOBReader),
// ReadableStreamBYOBRequest: nonEnumerable(streams.ReadableStreamBYOBRequest),
// ReadableStreamDefaultController: nonEnumerable(
// streams.ReadableStreamDefaultController
// ),
// TransformStream: nonEnumerable(streams.TransformStream),
// TransformStreamDefaultController: nonEnumerable(
// streams.TransformStreamDefaultController
// ),
// WritableStream: nonEnumerable(streams.WritableStream),
// WritableStreamDefaultWriter: nonEnumerable(
// streams.WritableStreamDefaultWriter
// ),
// WritableStreamDefaultController: nonEnumerable(
// streams.WritableStreamDefaultController
// ),
// // event
// CloseEvent: nonEnumerable(event.CloseEvent),
// CustomEvent: nonEnumerable(event.CustomEvent),
// ErrorEvent: nonEnumerable(event.ErrorEvent),
// Event: nonEnumerable(event.Event),
// EventTarget: nonEnumerable(event.EventTarget),
// MessageEvent: nonEnumerable(event.MessageEvent),
// PromiseRejectionEvent: nonEnumerable(event.PromiseRejectionEvent),
// ProgressEvent: nonEnumerable(event.ProgressEvent),
// reportError: writable(event.reportError),
// DOMException: nonEnumerable(DOMException),
// // file
// Blob: nonEnumerable(file.Blob),
// File: nonEnumerable(file.File),
// FileReader: nonEnumerable(fileReader.FileReader),
// // form data
// FormData: nonEnumerable(formData.FormData),
// // abort signal
// AbortController: nonEnumerable(abortSignal.AbortController),
// AbortSignal: nonEnumerable(abortSignal.AbortSignal),
// // // web sockets
// // WebSocket: nonEnumerable(webSocket.WebSocket),
// // // performance
// // Performance: nonEnumerable(performance.Performance),
// // PerformanceEntry: nonEnumerable(performance.PerformanceEntry),
// // PerformanceMark: nonEnumerable(performance.PerformanceMark),
// // PerformanceMeasure: nonEnumerable(performance.PerformanceMeasure),
// // performance: writable(performance.performance),
// // messagePort
// // structuredClone: writable(messagePort.structuredClone),
// // Branding as a WebIDL object
// [webidl.brand]: nonEnumerable(webidl.brand),
// });
// function nonEnumerable(value) {
// return {
// value,
// writable: true,
// enumerable: false,
// configurable: true,
// };
// }
// function writable(value) {
// return {
// value,
// writable: true,
// enumerable: true,
// configurable: true,
// };
// }
// function readOnly(value) {
// return {
// value,
// enumerable: true,
// writable: false,
// configurable: true,
// };
// }
// function getterOnly(getter) {
// return {
// get: getter,
// set() {},
// enumerable: true,
// configurable: true,
// };
// }
// function formatException(error) {
// if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) {
// return null;
// } else if (typeof error == "string") {
// return `Uncaught ${console.inspectArgs([console.quoteString(error)], {
// colors: false,
// })}`;
// } else {
// return `Uncaught ${console.inspectArgs([error], { colors: false })}`;
// }
// }