Compare commits
4 Commits
draft-fail
...
remove-den
| Author | SHA1 | Date | |
|---|---|---|---|
|
58cbb615e5 | ||
|
|
e12186cef2 | ||
|
|
095105ed24 | ||
|
|
13c55c61e8 |
@@ -0,0 +1,288 @@
|
||||
|
||||
▐▛███▜▌ Claude Code v2.1.23
|
||||
▝▜█████▛▘ Opus 4.5 · Claude Max
|
||||
▘▘ ▝▝ ~/windmill/backend
|
||||
|
||||
✻ Conversation compacted (ctrl+o for history)
|
||||
|
||||
|
||||
● Let me read the current state of the test file to understand the compilation
|
||||
error and fix it.
|
||||
|
||||
● Read 1 file (ctrl+o to expand)
|
||||
|
||||
● Let me check the common test module to understand the run_until_complete
|
||||
implementation and the Send trait issue.
|
||||
|
||||
● Read 1 file (ctrl+o to expand)
|
||||
|
||||
● Now I understand the issue. The run_until_complete function uses non-Send
|
||||
types internally (like PgListener), so it can't be spawned with tokio::spawn.
|
||||
The correct approach for stress testing is to push all jobs to the queue
|
||||
first, then let a single worker process them concurrently. This is actually
|
||||
more realistic since it tests the actual concurrent V8 execution.
|
||||
|
||||
Let me fix the stress tests:
|
||||
|
||||
● Update(tests/nativets_tests.rs)
|
||||
⎿ Added 88 lines, removed 88 lines
|
||||
743 // Stress Tests - Concurrent Execution
|
||||
744 // ===========================================================
|
||||
=================
|
||||
745
|
||||
746 +/// Push multiple jobs and process them with a single worker.
|
||||
747 +/// This tests concurrent V8 isolate execution more realistica
|
||||
+lly.
|
||||
748 +async fn push_jobs_and_wait_for_completion(
|
||||
749 + db: &Pool<Postgres>,
|
||||
750 + payloads: Vec<JobPayload>,
|
||||
751 + port: u16,
|
||||
752 +) -> Vec<CompletedJob> {
|
||||
753 + use windmill_common::worker::Connection;
|
||||
754 +
|
||||
755 + // Push all jobs first
|
||||
756 + let mut job_ids = Vec::with_capacity(payloads.len());
|
||||
757 + for payload in payloads {
|
||||
758 + let uuid = RunJob::from(payload).push(db).await;
|
||||
759 + job_ids.push(uuid);
|
||||
760 + }
|
||||
761 +
|
||||
762 + // Listen for completed jobs
|
||||
763 + let mut listener = listen_for_completed_jobs(db).await;
|
||||
764 +
|
||||
765 + // Start the worker and wait for all jobs to complete
|
||||
766 + let job_count = job_ids.len();
|
||||
767 + let db_clone = db.clone();
|
||||
768 + let job_ids_clone = job_ids.clone();
|
||||
769 +
|
||||
770 + in_test_worker(
|
||||
771 + Connection::Sql(db.clone()),
|
||||
772 + async move {
|
||||
773 + let mut completed_count = 0;
|
||||
774 + while completed_count < job_count {
|
||||
775 + if let Some(uuid) = listener.next().await {
|
||||
776 + if job_ids_clone.contains(&uuid) {
|
||||
777 + completed_count += 1;
|
||||
778 + }
|
||||
779 + }
|
||||
780 + }
|
||||
781 + },
|
||||
782 + port,
|
||||
783 + )
|
||||
784 + .await;
|
||||
785 +
|
||||
786 + // Fetch all completed jobs
|
||||
787 + let mut results = Vec::with_capacity(job_ids.len());
|
||||
788 + for uuid in job_ids {
|
||||
789 + results.push(completed_job(uuid, db).await);
|
||||
790 + }
|
||||
791 + results
|
||||
792 +}
|
||||
793 +
|
||||
794 #[sqlx::test(fixtures("base"))]
|
||||
795 async fn test_stress_concurrent_fetch(db: Pool<Postgres>) -> a
|
||||
nyhow::Result<()> {
|
||||
796 initialize_tracing().await;
|
||||
...
|
||||
806
|
||||
807 const NUM_CONCURRENT: usize = 20;
|
||||
808
|
||||
761 - let barrier = Arc::new(Barrier::new(NUM_CONCURRENT));
|
||||
762 - let mut handles = Vec::new();
|
||||
763 -
|
||||
809 + // Create all job payloads
|
||||
810 + let mut payloads = Vec::with_capacity(NUM_CONCURRENT);
|
||||
811 for i in 0..NUM_CONCURRENT {
|
||||
765 - let db = db.clone();
|
||||
766 - let mock_url = mock.url("/counter/increment");
|
||||
767 - let barrier = barrier.clone();
|
||||
768 -
|
||||
769 - let handle = tokio::spawn(async move {
|
||||
770 - // Wait for all tasks to be ready
|
||||
771 - barrier.wait().await;
|
||||
772 -
|
||||
773 - let code = format!(r#"
|
||||
812 + let code = format!(r#"
|
||||
813 export async function main() {{
|
||||
814 const response = await fetch("{}", {{ method: "POST" }});
|
||||
776 - return await response.json();
|
||||
815 + const data = await response.json();
|
||||
816 + return {{ ...data, jobIndex: {} }};
|
||||
817 }}
|
||||
778 -"#, mock_url);
|
||||
779 -
|
||||
780 - let result = run_nativets(&db, &code, port).await;
|
||||
781 - (i, result.success, result.json_result())
|
||||
782 - });
|
||||
783 - handles.push(handle);
|
||||
818 +"#, mock.url("/counter/increment"), i);
|
||||
819 + payloads.push(nativets_code(&code));
|
||||
820 }
|
||||
821
|
||||
786 - // Wait for all jobs to complete
|
||||
787 - let results: Vec<_> = futures::future::join_all(handles)
|
||||
788 - .await
|
||||
789 - .into_iter()
|
||||
790 - .map(|r| r.unwrap())
|
||||
791 - .collect();
|
||||
822 + // Push all jobs and wait for completion
|
||||
823 + let results = push_jobs_and_wait_for_completion(&db, paylo
|
||||
+ads, port).await;
|
||||
824
|
||||
825 // Verify all succeeded
|
||||
794 - for (i, success, _) in &results {
|
||||
795 - assert!(success, "Job {} should succeed", i);
|
||||
826 + for (i, result) in results.iter().enumerate() {
|
||||
827 + assert!(result.success, "Job {} should succeed", i);
|
||||
828 }
|
||||
829
|
||||
830 // Verify counter was incremented NUM_CONCURRENT times
|
||||
...
|
||||
849
|
||||
850 const NUM_CONCURRENT: usize = 10;
|
||||
851
|
||||
820 - let barrier = Arc::new(Barrier::new(NUM_CONCURRENT));
|
||||
821 - let mut handles = Vec::new();
|
||||
822 -
|
||||
852 + // Create all job payloads with CPU-bound computation
|
||||
853 + let mut payloads = Vec::with_capacity(NUM_CONCURRENT);
|
||||
854 for i in 0..NUM_CONCURRENT {
|
||||
824 - let db = db.clone();
|
||||
825 - let barrier = barrier.clone();
|
||||
826 -
|
||||
827 - let handle = tokio::spawn(async move {
|
||||
828 - barrier.wait().await;
|
||||
829 -
|
||||
830 - // CPU-bound computation
|
||||
831 - let code = r#"
|
||||
832 -export async function main() {
|
||||
855 + let code = format!(r#"
|
||||
856 +export async function main() {{
|
||||
857 // Compute fibonacci iteratively
|
||||
834 - function fib(n: number): number {
|
||||
858 + function fib(n: number): number {{
|
||||
859 let [a, b] = [0, 1];
|
||||
836 - for (let i = 0; i < n; i++) {
|
||||
860 + for (let i = 0; i < n; i++) {{
|
||||
861 [a, b] = [b, a + b];
|
||||
838 - }
|
||||
862 + }}
|
||||
863 return a;
|
||||
840 - }
|
||||
864 + }}
|
||||
865
|
||||
866 // Do some work
|
||||
867 const results = [];
|
||||
844 - for (let i = 0; i < 100; i++) {
|
||||
868 + for (let i = 0; i < 100; i++) {{
|
||||
869 results.push(fib(20));
|
||||
846 - }
|
||||
870 + }}
|
||||
871
|
||||
848 - return { computed: results.length, sample: results[0] };
|
||||
849 -}
|
||||
850 -"#;
|
||||
851 -
|
||||
852 - let result = run_nativets(&db, code, port).await;
|
||||
853 - (i, result.success, result.json_result())
|
||||
854 - });
|
||||
855 - handles.push(handle);
|
||||
872 + return {{ computed: results.length, sample: results[0], jo
|
||||
+bIndex: {} }};
|
||||
873 +}}
|
||||
874 +"#, i);
|
||||
875 + payloads.push(nativets_code(&code));
|
||||
876 }
|
||||
877
|
||||
858 - let results: Vec<_> = futures::future::join_all(handles)
|
||||
859 - .await
|
||||
860 - .into_iter()
|
||||
861 - .map(|r| r.unwrap())
|
||||
862 - .collect();
|
||||
878 + // Push all jobs and wait for completion
|
||||
879 + let results = push_jobs_and_wait_for_completion(&db, paylo
|
||||
+ads, port).await;
|
||||
880
|
||||
864 - for (i, success, result) in &results {
|
||||
865 - assert!(success, "Job {} should succeed", i);
|
||||
866 - let json = result.as_ref().unwrap();
|
||||
881 + for (i, result) in results.iter().enumerate() {
|
||||
882 + assert!(result.success, "Job {} should succeed", i);
|
||||
883 + let json = result.json_result().unwrap();
|
||||
884 assert_eq!(json["computed"], 100);
|
||||
885 assert_eq!(json["sample"], 6765); // fib(20)
|
||||
886 }
|
||||
...
|
||||
897
|
||||
898 const NUM_CONCURRENT: usize = 15;
|
||||
899
|
||||
883 - let barrier = Arc::new(Barrier::new(NUM_CONCURRENT));
|
||||
884 - let mut handles = Vec::new();
|
||||
885 -
|
||||
900 + // Create all job payloads with mixed workload
|
||||
901 + let mut payloads = Vec::with_capacity(NUM_CONCURRENT);
|
||||
902 for i in 0..NUM_CONCURRENT {
|
||||
887 - let db = db.clone();
|
||||
888 - let mock_url = mock.url("");
|
||||
889 - let barrier = barrier.clone();
|
||||
890 -
|
||||
891 - let handle = tokio::spawn(async move {
|
||||
892 - barrier.wait().await;
|
||||
893 -
|
||||
894 - // Mixed workload: fetch + computation
|
||||
895 - let code = format!(r#"
|
||||
903 + let code = format!(r#"
|
||||
904 export async function main() {{
|
||||
905 // Fetch data
|
||||
906 const response = await fetch("{}/array/50");
|
||||
...
|
||||
928 jobIndex: {}
|
||||
929 }};
|
||||
930 }}
|
||||
923 -"#, mock_url, i);
|
||||
924 -
|
||||
925 - let result = run_nativets(&db, &code, port).await;
|
||||
926 - (i, result.success, result.json_result())
|
||||
927 - });
|
||||
928 - handles.push(handle);
|
||||
931 +"#, mock.url(""), i);
|
||||
932 + payloads.push(nativets_code(&code));
|
||||
933 }
|
||||
934
|
||||
931 - let results: Vec<_> = futures::future::join_all(handles)
|
||||
932 - .await
|
||||
933 - .into_iter()
|
||||
934 - .map(|r| r.unwrap())
|
||||
935 - .collect();
|
||||
935 + // Push all jobs and wait for completion
|
||||
936 + let results = push_jobs_and_wait_for_completion(&db, paylo
|
||||
+ads, port).await;
|
||||
937
|
||||
937 - for (i, success, result) in &results {
|
||||
938 - assert!(success, "Job {} should succeed", i);
|
||||
939 - let json = result.as_ref().unwrap();
|
||||
938 + for result in &results {
|
||||
939 + assert!(result.success, "Job should succeed");
|
||||
940 + let json = result.json_result().unwrap();
|
||||
941 assert_eq!(json["primeCount"], 25); // Primes under 10
|
||||
0
|
||||
941 - assert_eq!(json["jobIndex"], *i);
|
||||
942 }
|
||||
943
|
||||
944 mock.shutdown().await;
|
||||
|
||||
● Now let me update the imports to remove unused ones and add the required ones:
|
||||
|
||||
● Update(tests/nativets_tests.rs)
|
||||
⎿ Removed 3 lines
|
||||
21 use sqlx::{Pool, Postgres};
|
||||
22 use std::net::SocketAddr;
|
||||
23 use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
24 -use std::sync::Arc;
|
||||
25 -use tokio::sync::Barrier;
|
||||
26 -use uuid::Uuid;
|
||||
24 use windmill_common::jobs::{JobPayload, RawCode};
|
||||
25 use windmill_common::scripts::ScriptLang;
|
||||
26
|
||||
|
||||
● Let me compile the test to check for any remaining issues:
|
||||
57
backend/Cargo.lock
generated
57
backend/Cargo.lock
generated
@@ -15521,6 +15521,7 @@ dependencies = [
|
||||
"windmill-common",
|
||||
"windmill-git-sync",
|
||||
"windmill-indexer",
|
||||
"windmill-nativets",
|
||||
"windmill-queue",
|
||||
"windmill-worker",
|
||||
"windows-service",
|
||||
@@ -15566,8 +15567,6 @@ dependencies = [
|
||||
"cookie 0.17.0",
|
||||
"cron",
|
||||
"datafusion",
|
||||
"deno_core",
|
||||
"deno_error",
|
||||
"ed25519-dalek",
|
||||
"flate2",
|
||||
"futures",
|
||||
@@ -15604,6 +15603,7 @@ dependencies = [
|
||||
"rdkafka-sys",
|
||||
"regex",
|
||||
"reqwest 0.13.1",
|
||||
"rquickjs",
|
||||
"rsa",
|
||||
"rumqttc",
|
||||
"rust-embed",
|
||||
@@ -15871,6 +15871,43 @@ dependencies = [
|
||||
"windmill-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-nativets"
|
||||
version = "1.621.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"const_format",
|
||||
"crossbeam-channel",
|
||||
"deno_ast",
|
||||
"deno_console",
|
||||
"deno_core",
|
||||
"deno_error",
|
||||
"deno_fetch",
|
||||
"deno_io",
|
||||
"deno_net",
|
||||
"deno_permissions",
|
||||
"deno_runtime",
|
||||
"deno_telemetry",
|
||||
"deno_tls",
|
||||
"deno_url",
|
||||
"deno_web",
|
||||
"deno_webidl",
|
||||
"futures",
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlx",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"winapi",
|
||||
"windmill-common",
|
||||
"windmill-parser-ts",
|
||||
"windmill-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-oauth"
|
||||
version = "1.621.1"
|
||||
@@ -16167,20 +16204,6 @@ dependencies = [
|
||||
"chrono",
|
||||
"const_format",
|
||||
"convert_case 0.6.0",
|
||||
"deno_ast",
|
||||
"deno_console",
|
||||
"deno_core",
|
||||
"deno_error",
|
||||
"deno_fetch",
|
||||
"deno_io",
|
||||
"deno_net",
|
||||
"deno_permissions",
|
||||
"deno_runtime",
|
||||
"deno_telemetry",
|
||||
"deno_tls",
|
||||
"deno_url",
|
||||
"deno_web",
|
||||
"deno_webidl",
|
||||
"derive_more 1.0.0",
|
||||
"dotenv",
|
||||
"eventsource-stream",
|
||||
@@ -16221,7 +16244,6 @@ dependencies = [
|
||||
"reqwest-middleware",
|
||||
"rquickjs",
|
||||
"rust_decimal",
|
||||
"rustls-pemfile 2.2.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
@@ -16244,6 +16266,7 @@ dependencies = [
|
||||
"windmill-git-sync",
|
||||
"windmill-macros",
|
||||
"windmill-mcp",
|
||||
"windmill-nativets",
|
||||
"windmill-parser",
|
||||
"windmill-parser-bash",
|
||||
"windmill-parser-csharp",
|
||||
|
||||
@@ -18,6 +18,7 @@ members = [
|
||||
"./windmill-indexer",
|
||||
"./windmill-macros",
|
||||
"./windmill-oauth",
|
||||
"./windmill-nativets",
|
||||
"./parsers/windmill-parser",
|
||||
"./parsers/windmill-parser-ts",
|
||||
"./parsers/windmill-parser-go",
|
||||
@@ -67,8 +68,8 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud
|
||||
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
|
||||
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
|
||||
sqlx = ["windmill-worker/sqlx"]
|
||||
deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"]
|
||||
quickjs = ["windmill-worker/quickjs"]
|
||||
# deno_core is only needed for NativeTS execution - flow expressions and batch rerun use QuickJS (always enabled)
|
||||
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8", "dep:windmill-nativets"]
|
||||
deno_core_mac = ["deno_core", "windmill-worker/libffi_mac"]
|
||||
kafka = ["windmill-api/kafka"]
|
||||
nats = ["windmill-api/nats"]
|
||||
@@ -146,6 +147,7 @@ serde_derive.workspace = true
|
||||
serde_yml.workspace = true
|
||||
serde.workspace = true
|
||||
deno_core = { workspace = true, optional = true }
|
||||
windmill-nativets = { workspace = true, optional = true }
|
||||
object_store = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
constant_time_eq = { workspace = true, optional = true }
|
||||
@@ -196,6 +198,7 @@ windmill-indexer = {path = "./windmill-indexer"}
|
||||
windmill-mcp = {path = "./windmill-mcp"}
|
||||
windmill-oauth = {path = "./windmill-oauth"}
|
||||
windmill-macros = {path = "./windmill-macros"}
|
||||
windmill-nativets = {path = "./windmill-nativets"}
|
||||
windmill-parser = { path = "./parsers/windmill-parser" }
|
||||
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
|
||||
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
|
||||
@@ -280,7 +283,7 @@ aws-sigv4 = "^1.3.4"
|
||||
aws-sdk-config = "=1.68.0"
|
||||
aws-sdk-rds = "^1"
|
||||
async-trait = "0.1.88"
|
||||
|
||||
crossbeam-channel = "0.5"
|
||||
|
||||
v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix
|
||||
deno_fetch = "0.214.0"
|
||||
|
||||
@@ -1762,6 +1762,12 @@ pub async fn run_workers(
|
||||
*windmill_worker::SLEEP_QUEUE
|
||||
);
|
||||
|
||||
// Initialize NativeTS isolate pool if enabled (deno_core feature)
|
||||
#[cfg(feature = "deno_core")]
|
||||
{
|
||||
windmill_nativets::init_isolate_pool();
|
||||
}
|
||||
|
||||
for i in 1..(num_workers + 1) {
|
||||
let wk_conf = &workers[i as usize - 1];
|
||||
let conn1 = wk_conf.conn.clone();
|
||||
|
||||
1336
backend/tests/nativets_tests.rs
Normal file
1336
backend/tests/nativets_tests.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -34,7 +34,6 @@ postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:
|
||||
mqtt_trigger = ["dep:thiserror", "dep:rumqttc"]
|
||||
native_trigger = ["dep:strum", "dep:backon", "oauth2"]
|
||||
sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config"]
|
||||
deno_core = ["dep:deno_core", "dep:deno_error"]
|
||||
gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
|
||||
cloud = ["windmill-common/cloud"]
|
||||
mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"]
|
||||
@@ -162,12 +161,9 @@ async-trait.workspace = true
|
||||
google-cloud-pubsub = { workspace = true, optional = true }
|
||||
google-cloud-googleapis = { workspace = true , optional = true }
|
||||
tonic = { workspace = true, optional = true }
|
||||
deno_error = { workspace = true, optional = true }
|
||||
deno_core = { workspace = true, optional = true }
|
||||
# rquickjs is required for batch rerun JavaScript expression evaluation
|
||||
rquickjs.workspace = true
|
||||
tar.workspace = true
|
||||
flate2.workspace = true
|
||||
backon = {workspace = true, optional = true}
|
||||
strum = { workspace = true, optional = true }
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = { workspace = true, optional = true }
|
||||
|
||||
@@ -9,8 +9,7 @@
|
||||
use axum::body::Body;
|
||||
use axum::extract::Request;
|
||||
use axum::http::HeaderValue;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use deno_core::{op2, serde_v8, v8, JsRuntime, OpState};
|
||||
use rquickjs::{async_with, AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object};
|
||||
use futures::{StreamExt, TryFutureExt};
|
||||
use http::{HeaderMap, HeaderName};
|
||||
use itertools::Itertools;
|
||||
@@ -3921,50 +3920,159 @@ struct BatchReRunQueryReturnType {
|
||||
schema: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
#[op2]
|
||||
#[string]
|
||||
fn get_deno_core_job_value(state: &mut OpState) -> Option<String> {
|
||||
let obj = state.borrow::<BatchReRunQueryReturnType>();
|
||||
let str = serde_json::to_string(&obj).ok()?;
|
||||
Some(str)
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
/// Evaluate a JavaScript expression for batch rerun input transformation using QuickJS
|
||||
async fn batch_rerun_compute_js_expression(
|
||||
expr: String,
|
||||
job: BatchReRunQueryReturnType,
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let ext = deno_core::Extension {
|
||||
name: "batch_rerun_arg_transform_ext",
|
||||
ops: vec![get_deno_core_job_value()].into(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut isolate =
|
||||
JsRuntime::new(deno_core::RuntimeOptions { extensions: vec![ext], ..Default::default() });
|
||||
// Clone job without schema to match the original behavior
|
||||
let job_for_eval = BatchReRunQueryReturnType { schema: None, ..job };
|
||||
|
||||
{
|
||||
let op_state = isolate.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(BatchReRunQueryReturnType { schema: None, ..job });
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(10000),
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
rt.block_on(async move {
|
||||
let runtime = AsyncRuntime::new()
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let context = AsyncContext::full(&runtime).await
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
async_with!(context => |ctx| {
|
||||
let globals = ctx.globals();
|
||||
|
||||
// Set up the job object
|
||||
let job_json = serde_json::to_value(&job_for_eval)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let job_js = json_to_js_value(&ctx, &job_json)?;
|
||||
globals.set("job", job_js)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to set job: {}", e)))?;
|
||||
|
||||
// Evaluate the expression
|
||||
let result: rquickjs::Value = ctx.eval(expr.as_bytes())
|
||||
.catch(&ctx)
|
||||
.map_err(|e| Error::ExecutionErr(format!("QuickJS evaluation error: {}", e)))?;
|
||||
|
||||
// Convert result to JSON
|
||||
let json_result = js_to_json_value(&ctx, &result)?;
|
||||
let json_str = serde_json::to_string(&json_result)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let raw_value = JsonRawValue::from_string(json_str)?;
|
||||
Ok(raw_value)
|
||||
})
|
||||
.await
|
||||
})
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::ExecutionErr("Batch rerun expression evaluation timed out (>10s)".to_string()))?
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
}
|
||||
|
||||
/// Convert a serde_json::Value to a QuickJS Value
|
||||
fn json_to_js_value<'js>(
|
||||
ctx: &rquickjs::Ctx<'js>,
|
||||
val: &serde_json::Value,
|
||||
) -> error::Result<rquickjs::Value<'js>> {
|
||||
match val {
|
||||
serde_json::Value::Null => Ok(rquickjs::Value::new_null(ctx.clone())),
|
||||
serde_json::Value::Bool(b) => Ok(rquickjs::Value::new_bool(ctx.clone(), *b)),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(i) = n.as_i64() {
|
||||
if i >= i32::MIN as i64 && i <= i32::MAX as i64 {
|
||||
Ok(rquickjs::Value::new_int(ctx.clone(), i as i32))
|
||||
} else {
|
||||
Ok(rquickjs::Value::new_float(ctx.clone(), i as f64))
|
||||
}
|
||||
} else if let Some(f) = n.as_f64() {
|
||||
Ok(rquickjs::Value::new_float(ctx.clone(), f))
|
||||
} else {
|
||||
Ok(rquickjs::Value::new_float(ctx.clone(), 0.0))
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => s.clone().into_js(ctx)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to convert string: {}", e))),
|
||||
serde_json::Value::Array(arr) => {
|
||||
let js_arr = rquickjs::Array::new(ctx.clone())
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to create array: {}", e)))?;
|
||||
for (i, item) in arr.iter().enumerate() {
|
||||
js_arr.set(i, json_to_js_value(ctx, item)?)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to set array item: {}", e)))?;
|
||||
}
|
||||
Ok(js_arr.into_value())
|
||||
}
|
||||
serde_json::Value::Object(obj) => {
|
||||
let js_obj = Object::new(ctx.clone())
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to create object: {}", e)))?;
|
||||
for (k, v) in obj {
|
||||
js_obj.set(k.as_str(), json_to_js_value(ctx, v)?)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Failed to set object property: {}", e)))?;
|
||||
}
|
||||
Ok(js_obj.into_value())
|
||||
}
|
||||
}
|
||||
isolate
|
||||
.execute_script(
|
||||
"<batch_rerun_arg_transform>",
|
||||
"let job = JSON.parse(Deno.core.ops.get_deno_core_job_value());",
|
||||
)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Run user expr
|
||||
let result = isolate
|
||||
.execute_script("<batch_rerun_arg_transform>", expr)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let mut scope = isolate.handle_scope();
|
||||
let result = v8::Local::new(&mut scope, result);
|
||||
let result: serde_json::Value =
|
||||
serde_v8::from_v8(&mut scope, result).map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let result = JsonRawValue::from_string(result.to_string())?;
|
||||
Ok(result)
|
||||
/// Convert a QuickJS Value to a serde_json::Value
|
||||
fn js_to_json_value<'js>(
|
||||
ctx: &rquickjs::Ctx<'js>,
|
||||
val: &rquickjs::Value<'js>,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
if val.is_null() || val.is_undefined() {
|
||||
return Ok(serde_json::Value::Null);
|
||||
}
|
||||
|
||||
if let Some(b) = val.as_bool() {
|
||||
return Ok(serde_json::Value::Bool(b));
|
||||
}
|
||||
|
||||
if let Some(i) = val.as_int() {
|
||||
return Ok(serde_json::Value::Number(i.into()));
|
||||
}
|
||||
|
||||
if let Some(f) = val.as_float() {
|
||||
if f.fract() == 0.0 && f.abs() <= (i64::MAX as f64) {
|
||||
let i = f as i64;
|
||||
if (i as f64) == f {
|
||||
return Ok(serde_json::Value::Number(i.into()));
|
||||
}
|
||||
}
|
||||
if let Some(n) = serde_json::Number::from_f64(f) {
|
||||
return Ok(serde_json::Value::Number(n));
|
||||
} else {
|
||||
return Ok(serde_json::Value::Null);
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(s) = String::from_js(ctx, val.clone()) {
|
||||
return Ok(serde_json::Value::String(s));
|
||||
}
|
||||
|
||||
if let Ok(arr) = rquickjs::Array::from_js(ctx, val.clone()) {
|
||||
let mut json_arr = Vec::new();
|
||||
for i in 0..arr.len() {
|
||||
if let Ok(item) = arr.get::<rquickjs::Value>(i) {
|
||||
json_arr.push(js_to_json_value(ctx, &item)?);
|
||||
}
|
||||
}
|
||||
return Ok(serde_json::Value::Array(json_arr));
|
||||
}
|
||||
|
||||
if let Ok(obj) = Object::from_js(ctx, val.clone()) {
|
||||
let mut json_obj = serde_json::Map::new();
|
||||
for res in obj.props::<String, rquickjs::Value>() {
|
||||
if let Ok((k, v)) = res {
|
||||
json_obj.insert(k, js_to_json_value(ctx, &v)?);
|
||||
}
|
||||
}
|
||||
return Ok(serde_json::Value::Object(json_obj));
|
||||
}
|
||||
|
||||
Ok(serde_json::Value::String("[object]".to_string()))
|
||||
}
|
||||
|
||||
async fn batch_rerun_jobs(
|
||||
@@ -4091,13 +4199,6 @@ async fn batch_rerun_handle_job(
|
||||
args.insert(property_name.clone(), value.clone());
|
||||
}
|
||||
InputTransform::Javascript { expr } => {
|
||||
#[cfg(not(feature = "deno_core"))]
|
||||
Err(error::Error::ExecutionErr(
|
||||
format!("deno_core feature is not activated, cannot evaluate: {expr}")
|
||||
.to_string(),
|
||||
))?;
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
args.insert(
|
||||
property_name.clone(),
|
||||
batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?,
|
||||
|
||||
65
backend/windmill-nativets/Cargo.toml
Normal file
65
backend/windmill-nativets/Cargo.toml
Normal file
@@ -0,0 +1,65 @@
|
||||
[package]
|
||||
name = "windmill-nativets"
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "windmill_nativets"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
enterprise = ["windmill-common/enterprise"]
|
||||
private = []
|
||||
|
||||
[dependencies]
|
||||
windmill-common = { workspace = true, default-features = false }
|
||||
windmill-parser-ts.workspace = true
|
||||
windmill-queue.workspace = true
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
anyhow.workspace = true
|
||||
futures.workspace = true
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
itertools.workspace = true
|
||||
lazy_static.workspace = true
|
||||
regex.workspace = true
|
||||
const_format.workspace = true
|
||||
sqlx.workspace = true
|
||||
crossbeam-channel.workspace = true
|
||||
|
||||
# deno_core dependencies - these are what we're isolating
|
||||
deno_telemetry.workspace = true
|
||||
deno_fetch.workspace = true
|
||||
deno_webidl.workspace = true
|
||||
deno_web.workspace = true
|
||||
deno_net.workspace = true
|
||||
deno_console.workspace = true
|
||||
deno_url.workspace = true
|
||||
deno_core.workspace = true
|
||||
deno_ast.workspace = true
|
||||
deno_tls.workspace = true
|
||||
deno_permissions.workspace = true
|
||||
deno_io.workspace = true
|
||||
deno_error.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_fetch.workspace = true
|
||||
deno_webidl.workspace = true
|
||||
deno_web.workspace = true
|
||||
deno_net.workspace = true
|
||||
deno_console.workspace = true
|
||||
deno_url.workspace = true
|
||||
deno_core.workspace = true
|
||||
deno_ast.workspace = true
|
||||
deno_tls.workspace = true
|
||||
deno_permissions.workspace = true
|
||||
deno_io.workspace = true
|
||||
deno_runtime.workspace = true
|
||||
deno_telemetry.workspace = true
|
||||
|
||||
[target.'cfg(windows)'.build-dependencies]
|
||||
winapi.workspace = true
|
||||
124
backend/windmill-nativets/build.rs
Normal file
124
backend/windmill-nativets/build.rs
Normal file
@@ -0,0 +1,124 @@
|
||||
use deno_fetch::FetchPermissions;
|
||||
use deno_net::NetPermissions;
|
||||
use deno_web::{BlobStore, TimersPermission};
|
||||
use std::borrow::Cow;
|
||||
use std::env;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct PermissionsContainer;
|
||||
|
||||
impl FetchPermissions for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
_resolved: bool,
|
||||
_p: &'a std::path::Path,
|
||||
_api_name: &str,
|
||||
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
}
|
||||
|
||||
impl TimersPermission for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl NetPermissions for PermissionsContainer {
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
_p: &'a str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write<'a>(
|
||||
&mut self,
|
||||
_p: &'a str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
_host: &(T, Option<u16>),
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write_path<'a>(
|
||||
&mut self,
|
||||
_: &'a Path,
|
||||
_: &str,
|
||||
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
deno_core::extension!(
|
||||
fetch,
|
||||
esm_entry_point = "ext:fetch/src/runtime.js",
|
||||
esm = ["src/runtime.js"],
|
||||
);
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
|
||||
let exts = vec![
|
||||
deno_telemetry::deno_telemetry::init_ops_and_esm(),
|
||||
deno_webidl::deno_webidl::init_ops_and_esm(),
|
||||
deno_url::deno_url::init_ops_and_esm(),
|
||||
deno_console::deno_console::init_ops_and_esm(),
|
||||
deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
|
||||
Arc::new(BlobStore::default()),
|
||||
None,
|
||||
),
|
||||
deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
|
||||
deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(None, None),
|
||||
fetch::init_ops_and_esm(),
|
||||
];
|
||||
|
||||
// Build the file path to the snapshot.
|
||||
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
|
||||
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
|
||||
|
||||
// Create the snapshot.
|
||||
let output = deno_core::snapshot::create_snapshot(
|
||||
deno_core::snapshot::CreateSnapshotOptions {
|
||||
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
|
||||
startup_snapshot: None,
|
||||
extension_transpiler: Some(std::rc::Rc::new(|specifier, source| {
|
||||
deno_runtime::transpile::maybe_transpile_source(specifier, source)
|
||||
})),
|
||||
extensions: exts,
|
||||
with_runtime_cb: None,
|
||||
skip_op_registration: false,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut file = std::fs::File::create(snapshot_path).unwrap();
|
||||
file.write_all(&output.output).unwrap();
|
||||
|
||||
for path in output.files_loaded_during_snapshot {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
1002
backend/windmill-nativets/src/lib.rs
Normal file
1002
backend/windmill-nativets/src/lib.rs
Normal file
File diff suppressed because it is too large
Load Diff
212
backend/windmill-nativets/src/runtime.js
Normal file
212
backend/windmill-nativets/src/runtime.js
Normal file
@@ -0,0 +1,212 @@
|
||||
import * as abortSignal from "ext:deno_web/03_abort_signal.js";
|
||||
import * as base64 from "ext:deno_web/05_base64.js";
|
||||
import * as console from "ext:deno_console/01_console.js";
|
||||
import * as encoding from "ext:deno_web/08_text_encoding.js";
|
||||
import * as event from "ext:deno_web/02_event.js";
|
||||
import * as fetch from "ext:deno_fetch/26_fetch.js";
|
||||
import * as file from "ext:deno_web/09_file.js";
|
||||
import * as fileReader from "ext:deno_web/10_filereader.js";
|
||||
import * as formData from "ext:deno_fetch/21_formdata.js";
|
||||
import * as headers from "ext:deno_fetch/20_headers.js";
|
||||
import * as streams from "ext:deno_web/06_streams.js";
|
||||
import * as timers from "ext:deno_web/02_timers.js";
|
||||
import * as url from "ext:deno_url/00_url.js";
|
||||
import * as net from "ext:deno_net/01_net.js";
|
||||
import * as tls from "ext:deno_net/02_tls.js";
|
||||
import * as urlPattern from "ext:deno_url/01_urlpattern.js";
|
||||
import * as webidl from "ext:deno_webidl/00_webidl.js";
|
||||
import * as response from "ext:deno_fetch/23_response.js";
|
||||
import * as request from "ext:deno_fetch/23_request.js";
|
||||
import "ext:deno_web/02_structured_clone.js";
|
||||
import "ext:deno_web/04_global_interfaces.js";
|
||||
import "ext:deno_web/13_message_port.js";
|
||||
import "ext:deno_web/14_compression.js";
|
||||
import "ext:deno_web/15_performance.js";
|
||||
import "ext:deno_web/16_image_data.js";
|
||||
import "ext:deno_fetch/27_eventsource.js";
|
||||
|
||||
globalThis.atob = base64.atob;
|
||||
globalThis.btoa = base64.btoa;
|
||||
globalThis.fetch = fetch.fetch;
|
||||
globalThis.Request = request.Request;
|
||||
globalThis.Response = response.Response;
|
||||
globalThis.Blob = file.Blob;
|
||||
globalThis.URL = url.URL;
|
||||
globalThis.FormData = formData.FormData;
|
||||
globalThis.URLSearchParams = url.URLSearchParams;
|
||||
globalThis.Headers = headers.Headers;
|
||||
globalThis.FileReader = fileReader.FileReader;
|
||||
globalThis.console = new console.Console((msg, level) =>
|
||||
globalThis.Deno.core.ops.op_log(msg)
|
||||
);
|
||||
globalThis.AbortController = abortSignal.AbortController;
|
||||
globalThis.AbortSignal = abortSignal.AbortSignal;
|
||||
|
||||
Object.assign(globalThis, {
|
||||
clearInterval: timers.clearInterval,
|
||||
clearTimeout: timers.clearTimeout,
|
||||
setInterval: timers.setInterval,
|
||||
setTimeout: timers.setTimeout,
|
||||
});
|
||||
|
||||
// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
|
||||
// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
|
||||
// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
|
||||
// consoleConfig: 0=ignore, 1=capture, 2=replace
|
||||
globalThis.__bootstrapOtel = () => {
|
||||
import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => {
|
||||
bootstrap([1, 0, 1, 0]);
|
||||
// Expose enterSpan for setting parent trace context
|
||||
globalThis.__enterSpan = enterSpan;
|
||||
});
|
||||
};
|
||||
|
||||
// Object.assign(globalThis, {
|
||||
// console: nonEnumerable(
|
||||
// new console.Console((msg, level) => core.print(msg, level > 1))
|
||||
// ),
|
||||
|
||||
// // timers
|
||||
|
||||
// // fetch
|
||||
// Request: nonEnumerable(request.Request),
|
||||
// Response: nonEnumerable(response.Response),
|
||||
// Headers: nonEnumerable(headers.Headers),
|
||||
// fetch: writable(fetch.fetch),
|
||||
|
||||
// // base64
|
||||
// atob: writable(base64.atob),
|
||||
// btoa: writable(base64.btoa),
|
||||
|
||||
// // encoding
|
||||
// TextDecoder: nonEnumerable(encoding.TextDecoder),
|
||||
// TextEncoder: nonEnumerable(encoding.TextEncoder),
|
||||
// TextDecoderStream: nonEnumerable(encoding.TextDecoderStream),
|
||||
// TextEncoderStream: nonEnumerable(encoding.TextEncoderStream),
|
||||
|
||||
// // url
|
||||
// URL: nonEnumerable(url.URL),
|
||||
// URLPattern: nonEnumerable(urlPattern.URLPattern),
|
||||
// URLSearchParams: nonEnumerable(url.URLSearchParams),
|
||||
|
||||
// // // crypto
|
||||
// // CryptoKey: nonEnumerable(crypto.CryptoKey),
|
||||
// // crypto: readOnly(crypto.crypto),
|
||||
// // Crypto: nonEnumerable(crypto.Crypto),
|
||||
// // SubtleCrypto: nonEnumerable(crypto.SubtleCrypto),
|
||||
|
||||
// // streams
|
||||
// ByteLengthQueuingStrategy: nonEnumerable(streams.ByteLengthQueuingStrategy),
|
||||
// CountQueuingStrategy: nonEnumerable(streams.CountQueuingStrategy),
|
||||
// ReadableStream: nonEnumerable(streams.ReadableStream),
|
||||
// ReadableStreamDefaultReader: nonEnumerable(
|
||||
// streams.ReadableStreamDefaultReader
|
||||
// ),
|
||||
// ReadableByteStreamController: nonEnumerable(
|
||||
// streams.ReadableByteStreamController
|
||||
// ),
|
||||
// ReadableStreamBYOBReader: nonEnumerable(streams.ReadableStreamBYOBReader),
|
||||
// ReadableStreamBYOBRequest: nonEnumerable(streams.ReadableStreamBYOBRequest),
|
||||
// ReadableStreamDefaultController: nonEnumerable(
|
||||
// streams.ReadableStreamDefaultController
|
||||
// ),
|
||||
// TransformStream: nonEnumerable(streams.TransformStream),
|
||||
// TransformStreamDefaultController: nonEnumerable(
|
||||
// streams.TransformStreamDefaultController
|
||||
// ),
|
||||
// WritableStream: nonEnumerable(streams.WritableStream),
|
||||
// WritableStreamDefaultWriter: nonEnumerable(
|
||||
// streams.WritableStreamDefaultWriter
|
||||
// ),
|
||||
// WritableStreamDefaultController: nonEnumerable(
|
||||
// streams.WritableStreamDefaultController
|
||||
// ),
|
||||
|
||||
// // event
|
||||
// CloseEvent: nonEnumerable(event.CloseEvent),
|
||||
// CustomEvent: nonEnumerable(event.CustomEvent),
|
||||
// ErrorEvent: nonEnumerable(event.ErrorEvent),
|
||||
// Event: nonEnumerable(event.Event),
|
||||
// EventTarget: nonEnumerable(event.EventTarget),
|
||||
// MessageEvent: nonEnumerable(event.MessageEvent),
|
||||
// PromiseRejectionEvent: nonEnumerable(event.PromiseRejectionEvent),
|
||||
// ProgressEvent: nonEnumerable(event.ProgressEvent),
|
||||
// reportError: writable(event.reportError),
|
||||
// DOMException: nonEnumerable(DOMException),
|
||||
|
||||
// // file
|
||||
// Blob: nonEnumerable(file.Blob),
|
||||
// File: nonEnumerable(file.File),
|
||||
// FileReader: nonEnumerable(fileReader.FileReader),
|
||||
|
||||
// // form data
|
||||
// FormData: nonEnumerable(formData.FormData),
|
||||
|
||||
// // abort signal
|
||||
// AbortController: nonEnumerable(abortSignal.AbortController),
|
||||
// AbortSignal: nonEnumerable(abortSignal.AbortSignal),
|
||||
|
||||
// // // web sockets
|
||||
// // WebSocket: nonEnumerable(webSocket.WebSocket),
|
||||
|
||||
// // // performance
|
||||
// // Performance: nonEnumerable(performance.Performance),
|
||||
// // PerformanceEntry: nonEnumerable(performance.PerformanceEntry),
|
||||
// // PerformanceMark: nonEnumerable(performance.PerformanceMark),
|
||||
// // PerformanceMeasure: nonEnumerable(performance.PerformanceMeasure),
|
||||
// // performance: writable(performance.performance),
|
||||
|
||||
// // messagePort
|
||||
// // structuredClone: writable(messagePort.structuredClone),
|
||||
|
||||
// // Branding as a WebIDL object
|
||||
// [webidl.brand]: nonEnumerable(webidl.brand),
|
||||
// });
|
||||
|
||||
// function nonEnumerable(value) {
|
||||
// return {
|
||||
// value,
|
||||
// writable: true,
|
||||
// enumerable: false,
|
||||
// configurable: true,
|
||||
// };
|
||||
// }
|
||||
|
||||
// function writable(value) {
|
||||
// return {
|
||||
// value,
|
||||
// writable: true,
|
||||
// enumerable: true,
|
||||
// configurable: true,
|
||||
// };
|
||||
// }
|
||||
|
||||
// function readOnly(value) {
|
||||
// return {
|
||||
// value,
|
||||
// enumerable: true,
|
||||
// writable: false,
|
||||
// configurable: true,
|
||||
// };
|
||||
// }
|
||||
|
||||
// function getterOnly(getter) {
|
||||
// return {
|
||||
// get: getter,
|
||||
// set() {},
|
||||
// enumerable: true,
|
||||
// configurable: true,
|
||||
// };
|
||||
// }
|
||||
|
||||
// function formatException(error) {
|
||||
// if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) {
|
||||
// return null;
|
||||
// } else if (typeof error == "string") {
|
||||
// return `Uncaught ${console.inspectArgs([console.quoteString(error)], {
|
||||
// colors: false,
|
||||
// })}`;
|
||||
// } else {
|
||||
// return `Uncaught ${console.inspectArgs([error], { colors: false })}`;
|
||||
// }
|
||||
// }
|
||||
10389
backend/windmill-nativets/src/windmill-client.js
Normal file
10389
backend/windmill-nativets/src/windmill-client.js
Normal file
File diff suppressed because it is too large
Load Diff
@@ -21,9 +21,11 @@ parquet = ["windmill-common/parquet", "dep:object_store"]
|
||||
flow_testing = []
|
||||
cloud = []
|
||||
sqlx = []
|
||||
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
|
||||
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile",
|
||||
"quickjs"]
|
||||
# deno_core is only used for NativeTS (eval_fetch_timeout) execution
|
||||
# Flow expressions use QuickJS which is always enabled (rquickjs is not optional)
|
||||
# NativeTS execution is isolated in the windmill-nativets crate to keep V8/deno_core
|
||||
# transitive dependencies separate from the rest of windmill-worker
|
||||
deno_core = ["dep:windmill-nativets"]
|
||||
libffi_mac = ["dep:libffi-sys"]
|
||||
otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"]
|
||||
dind = ["dep:bollard"]
|
||||
@@ -37,7 +39,6 @@ nu = ["dep:windmill-parser-nu"]
|
||||
java = ["dep:windmill-parser-java"]
|
||||
ruby = ["dep:windmill-parser-ruby"]
|
||||
duckdb = ["dep:libloading"]
|
||||
quickjs = ["dep:rquickjs"]
|
||||
bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"]
|
||||
|
||||
[dependencies]
|
||||
@@ -45,6 +46,7 @@ windmill-queue.workspace = true
|
||||
windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker.
|
||||
windmill-common = { workspace = true, default-features = false }
|
||||
windmill-mcp = { workspace = true, optional = true }
|
||||
windmill-nativets = { workspace = true, optional = true }
|
||||
windmill-macros.workspace = true
|
||||
windmill-parser.workspace = true
|
||||
windmill-parser-ts.workspace = true
|
||||
@@ -94,20 +96,6 @@ once_cell.workspace = true
|
||||
tokio-postgres.workspace = true
|
||||
bit-vec.workspace = true
|
||||
url.workspace = true
|
||||
deno_telemetry = { workspace = true, optional = true }
|
||||
deno_fetch = { workspace = true, optional = true }
|
||||
deno_webidl = { workspace = true, optional = true }
|
||||
deno_web = { workspace = true, optional = true }
|
||||
deno_net = { workspace = true, optional = true }
|
||||
deno_console = { workspace = true, optional = true }
|
||||
deno_url = { workspace = true, optional = true }
|
||||
deno_core = { workspace = true, optional = true }
|
||||
deno_ast = { workspace = true, optional = true }
|
||||
deno_tls = { workspace = true, optional = true }
|
||||
deno_permissions = { workspace = true, optional = true }
|
||||
deno_io = { workspace = true, optional = true }
|
||||
deno_error = { workspace = true, optional = true }
|
||||
rustls-pemfile = { workspace = true, optional = true }
|
||||
async-stream.workspace = true
|
||||
|
||||
postgres-native-tls.workspace = true
|
||||
@@ -147,7 +135,8 @@ prost.workspace = true
|
||||
axum.workspace = true
|
||||
bollard = { workspace = true, optional = true }
|
||||
oracle = { workspace = true, optional = true }
|
||||
rquickjs = { workspace = true, optional = true }
|
||||
# rquickjs is required for flow expression evaluation
|
||||
rquickjs.workspace = true
|
||||
hudsucker.workspace = true
|
||||
hyper-http-proxy.workspace = true
|
||||
hyper-tls.workspace = true
|
||||
@@ -155,18 +144,4 @@ hyper-util.workspace = true
|
||||
rcgen.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_fetch = { workspace = true, optional = true }
|
||||
deno_webidl = { workspace = true, optional = true }
|
||||
deno_web = { workspace = true, optional = true }
|
||||
deno_net = { workspace = true, optional = true }
|
||||
deno_console = { workspace = true, optional = true }
|
||||
deno_url = { workspace = true, optional = true }
|
||||
deno_core = { workspace = true, optional = true }
|
||||
deno_ast = { workspace = true, optional = true }
|
||||
deno_tls = { workspace = true, optional = true }
|
||||
deno_permissions = { workspace = true, optional = true }
|
||||
deno_io = { workspace = true, optional = true }
|
||||
deno_runtime = { workspace = true, optional = true }
|
||||
deno_telemetry = { workspace = true, optional = true }
|
||||
winapi = { workspace = true, optional = true }
|
||||
libffi-sys = { workspace = true, optional = true }
|
||||
|
||||
@@ -1,154 +1,3 @@
|
||||
#[cfg(feature = "deno_core")]
|
||||
use deno_fetch::FetchPermissions;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use deno_net::NetPermissions;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use deno_web::{BlobStore, TimersPermission};
|
||||
#[cfg(feature = "deno_core")]
|
||||
use std::borrow::Cow;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use std::env;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use std::io::Write;
|
||||
#[cfg(feature = "deno_core")]
|
||||
use std::path::{Path, PathBuf};
|
||||
#[cfg(feature = "deno_core")]
|
||||
use std::sync::Arc;
|
||||
|
||||
// #[cfg(feature = "deno_core")]
|
||||
pub struct PermissionsContainer;
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
impl FetchPermissions for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn check_net_url(
|
||||
&mut self,
|
||||
_url: &deno_core::url::Url,
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
_resolved: bool,
|
||||
_p: &'a std::path::Path,
|
||||
_api_name: &str,
|
||||
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
impl TimersPermission for PermissionsContainer {
|
||||
#[inline(always)]
|
||||
fn allow_hrtime(&mut self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
impl NetPermissions for PermissionsContainer {
|
||||
fn check_read<'a>(
|
||||
&mut self,
|
||||
_p: &'a str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write<'a>(
|
||||
&mut self,
|
||||
_p: &'a str,
|
||||
_api_name: &str,
|
||||
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_net<T: AsRef<str>>(
|
||||
&mut self,
|
||||
_host: &(T, Option<u16>),
|
||||
_api_name: &str,
|
||||
) -> Result<(), deno_permissions::PermissionCheckError> {
|
||||
unreachable!("snapshotting")
|
||||
}
|
||||
|
||||
fn check_write_path<'a>(
|
||||
&mut self,
|
||||
_: &'a Path,
|
||||
_: &str,
|
||||
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
deno_core::extension!(
|
||||
fetch,
|
||||
esm_entry_point = "ext:fetch/src/runtime.js",
|
||||
esm = ["src/runtime.js"],
|
||||
);
|
||||
|
||||
#[cfg(feature = "deno_core")]
|
||||
fn main() {
|
||||
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
|
||||
|
||||
let exts = vec![
|
||||
deno_telemetry::deno_telemetry::init_ops_and_esm(),
|
||||
deno_webidl::deno_webidl::init_ops_and_esm(),
|
||||
deno_url::deno_url::init_ops_and_esm(),
|
||||
deno_console::deno_console::init_ops_and_esm(),
|
||||
deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
|
||||
Arc::new(BlobStore::default()),
|
||||
None,
|
||||
),
|
||||
deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
|
||||
deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(None, None),
|
||||
fetch::init_ops_and_esm(),
|
||||
];
|
||||
|
||||
// Build the file path to the snapshot.
|
||||
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
|
||||
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
|
||||
|
||||
// Create the snapshot.
|
||||
let output = deno_core::snapshot::create_snapshot(
|
||||
deno_core::snapshot::CreateSnapshotOptions {
|
||||
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
|
||||
startup_snapshot: None,
|
||||
extension_transpiler: Some(std::rc::Rc::new(|specifier, source| {
|
||||
deno_runtime::transpile::maybe_transpile_source(specifier, source)
|
||||
})),
|
||||
extensions: exts,
|
||||
with_runtime_cb: None,
|
||||
skip_op_registration: false,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// NOTE(bartlomieju): Compressing the TSC snapshot in debug build took
|
||||
// ~45s on M1 MacBook Pro; without compression it took ~1s.
|
||||
// Thus we're not using compressed snapshot, trading off
|
||||
// a lot of build time for some startup time in debug build.
|
||||
let mut file = std::fs::File::create(snapshot_path).unwrap();
|
||||
// if cfg!(debug_assertions) {
|
||||
file.write_all(&output.output).unwrap();
|
||||
// } else {
|
||||
// let mut vec = Vec::with_capacity(output.output.len());
|
||||
// vec.extend((output.output.len() as u32).to_le_bytes());
|
||||
// vec.extend_from_slice(
|
||||
// &zstd::bulk::compress(&output.output, 22).expect("snapshot compression failed"),
|
||||
// );
|
||||
// file.write_all(&vec).unwrap();
|
||||
// }
|
||||
|
||||
for path in output.files_loaded_during_snapshot {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "deno_core"))]
|
||||
// Build script for windmill-worker
|
||||
// NativeTS/deno_core snapshot generation is now handled by windmill-nativets crate
|
||||
fn main() {}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1640,7 +1640,6 @@ mod benchmark_tests {
|
||||
use windmill_common::worker::to_raw_value;
|
||||
|
||||
/// Benchmark QuickJS expression evaluation startup time
|
||||
#[cfg(feature = "quickjs")]
|
||||
#[tokio::test]
|
||||
async fn benchmark_quickjs_startup() -> anyhow::Result<()> {
|
||||
use crate::js_eval_quickjs::eval_timeout_quickjs;
|
||||
|
||||
@@ -42,7 +42,7 @@ pub mod job_logger;
|
||||
pub mod job_logger_ee;
|
||||
mod job_logger_oss;
|
||||
mod js_eval;
|
||||
#[cfg(feature = "quickjs")]
|
||||
// QuickJS is always enabled for flow expression evaluation
|
||||
pub mod js_eval_quickjs;
|
||||
#[cfg(test)]
|
||||
mod js_eval_parity_tests;
|
||||
|
||||
Reference in New Issue
Block a user