diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 0221194d0c..2625049141 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -19,7 +19,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -40,7 +40,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -74,7 +74,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend diff --git a/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json b/backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json similarity index 89% rename from backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json rename to backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json index 2ce0b8015a..1e49b8405c 100644 --- a/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json +++ b/backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5) \n RETURNING id\n ", + "query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id\n ", "describe": { "columns": [ { @@ -53,5 +53,5 @@ false ] }, - "hash": "5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0" + "hash": "6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451" } diff --git a/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json b/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json new file mode 100644 index 0000000000..923f1a1af5 --- /dev/null +++ b/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory WHERE workspace_id = $1 AND conversation_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9" +} diff --git a/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json b/backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json similarity index 52% rename from backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json rename to backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json index ed3665b584..2a627e06b4 100644 --- a/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json +++ b/backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n importer_path,\n importer_kind::text as \"importer_kind!\", -- sqlx thinks this is nullable somehow, so enfore with !\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ", + "query": "\n SELECT\n importer_path,\n importer_kind::text as \"importer_kind!\",\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map\n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708" + "hash": "dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9" } diff --git a/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json similarity index 76% rename from backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json rename to backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json index 43d54b5a1c..26cf59d375 100644 --- a/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json +++ b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_dependencies\n SET archived = true \n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ", + "query": "\n UPDATE workspace_dependencies\n SET archived = true\n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ", "describe": { "columns": [ { @@ -51,5 +51,5 @@ false ] }, - "hash": "37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b" + "hash": "f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c5b47bc8c5..b3b700c7df 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15707,6 +15707,7 @@ dependencies = [ "windmill-api-client", "windmill-autoscaling", "windmill-common", + "windmill-dep-map", "windmill-git-sync", "windmill-indexer", "windmill-queue", @@ -15810,6 +15811,7 @@ dependencies = [ "windmill-audit", "windmill-autoscaling", "windmill-common", + "windmill-dep-map", "windmill-git-sync", "windmill-indexer", "windmill-mcp", @@ -16062,6 +16064,25 @@ dependencies = [ "windmill-parser-ts", ] +[[package]] +name = "windmill-dep-map" +version = "1.628.3" +dependencies = [ + "chrono", + "itertools 0.14.0", + "lazy_static", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-common", + "windmill-parser-py-imports", + "windmill-parser-ts", + "windmill-queue", +] + [[package]] name = "windmill-git-sync" version = "1.628.3" @@ -16845,6 +16866,7 @@ dependencies = [ "winapi", "windmill-audit", "windmill-common", + "windmill-dep-map", "windmill-git-sync", "windmill-macros", "windmill-mcp", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 91ac3add3e..860303f5ac 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -26,6 +26,7 @@ members = [ "./windmill-store", "./windmill-queue", "./windmill-worker", + "./windmill-dep-map", "./windmill-common", "./windmill-mcp", "./windmill-audit", @@ -129,11 +130,12 @@ all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", " # For windows we have another set of languages enabled all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"] # Edition meta-features: shared groups +inline_preview = ["windmill-api/inline_preview"] oss_core = [ "embedding", "parquet", "openidconnect", "license", "http_trigger", "zip", "oauth2", "postgres_trigger", "mqtt_trigger", "websocket", "smtp", "native_trigger", - "static_frontend", "mcp", "bedrock" + "static_frontend", "mcp", "bedrock", "inline_preview" ] ce_core = ["oss_core", "private"] ee_core = [ @@ -224,6 +226,7 @@ tikv-jemalloc-ctl = { optional = true, workspace = true } serde_json.workspace = true reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } +windmill-dep-map.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true @@ -235,6 +238,7 @@ tempfile.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-dep-map = { path = "./windmill-dep-map" } windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1b4fbeea70..9c34cb9df3 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ca79631be6c311c5300863634c539593f805e51e \ No newline at end of file +a69c11d8279401ede1ad5b54e3678c3efb3d2381 \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 07a4c5069b..b048a4992a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -84,10 +84,11 @@ use windmill_common::{ use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; use windmill_worker::{ - handle_job_error, JobCompletedSender, OtelTracingProxySettings, SameWorkerSender, - BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, - NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, + result_processor::handle_job_error, JobCompletedSender, OtelTracingProxySettings, + SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, + KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, + OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, + POWERSHELL_REPO_URL, }; #[cfg(feature = "parquet")] @@ -2103,7 +2104,11 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b } else { let wc = WORKER_CONFIG.read().await; let config = config.unwrap(); - let has_dedicated = config.dedicated_worker.is_some() || config.dedicated_workers.as_ref().is_some_and(|dws| !dws.is_empty()); + let has_dedicated = config.dedicated_worker.is_some() + || config + .dedicated_workers + .as_ref() + .is_some_and(|dws| !dws.is_empty()); if *wc != config || has_dedicated { if kill_if_change { if has_dedicated diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index ba89de3019..83e7f9931a 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -684,8 +684,10 @@ pub async fn run_deployed_relative_imports( language, priority: None, apply_preprocessor: false, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), - debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), }) .push(&db2) .await; @@ -734,8 +736,10 @@ pub async fn run_preview_relative_imports( cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), - debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), })) .push(&db2) .await; diff --git a/backend/tests/workspace_dependencies.rs b/backend/tests/workspace_dependencies.rs index b41eb7c550..124e727e91 100644 --- a/backend/tests/workspace_dependencies.rs +++ b/backend/tests/workspace_dependencies.rs @@ -7,7 +7,7 @@ mod workspace_dependencies { use sqlx::{Pool, Postgres}; use tokio_stream::StreamExt; use windmill_common::scripts::ScriptLang; - use windmill_worker::workspace_dependencies::NewWorkspaceDependencies; + use windmill_dep_map::workspace_dependencies::NewWorkspaceDependencies; mod deps { pub const REQUIREMENTS_IN: &'static str = "tiny==0.1.3"; // pub const GO_MOD: &'static str = r##" diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 1a3b6ee1f3..5ef0e43461 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -11,36 +11,37 @@ path = "src/lib.rs" [features] default = [] private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise"] stripe = [] -agent_worker_server = [] +inline_preview = ["dep:windmill-worker"] +agent_worker_server = ["dep:windmill-worker"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] -prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker?/parquet"] +prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] -kafka = ["dep:windmill-trigger-kafka", "windmill-trigger-kafka/enterprise", "windmill-store/kafka"] +kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] -nats = ["dep:windmill-trigger-nats", "windmill-trigger-nats/enterprise", "windmill-store/nats"] -websocket = ["dep:windmill-trigger-websocket", "windmill-trigger-websocket/enterprise"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email", "windmill-trigger-email/enterprise"] +nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] +websocket = ["dep:windmill-trigger-websocket"] +smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email"] license = ["dep:rsa"] zip = ["dep:async_zip"] oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"] -http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-trigger-http/enterprise", "windmill-store/http_trigger"] +http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-store/http_trigger"] static_frontend = ["dep:rust-embed"] -postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-trigger-postgres/enterprise", "windmill-store/postgres_trigger"] -mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-trigger-mqtt/enterprise", "windmill-store/mqtt_trigger"] -native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "windmill-native-triggers/enterprise", "dep:strum", "oauth2"] -sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-trigger-sqs/enterprise", "windmill-store/sqs_trigger"] +postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-store/postgres_trigger"] +mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-store/mqtt_trigger"] +native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "dep:strum", "oauth2"] +sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-store/sqs_trigger"] deno_core = ["dep:deno_core", "dep:deno_error"] -gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-trigger-gcp/enterprise", "windmill-store/gcp_trigger"] +gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-store/gcp_trigger"] cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud"] mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"] bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"] -python = [] +python = ["windmill-dep-map/python"] no_auth = ["windmill-api-auth/no_auth", "windmill-store/no_auth"] [dependencies] @@ -61,7 +62,8 @@ windmill-parser-py-imports.workspace = true windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling.workspace = true -windmill-worker.workspace = true +windmill-worker = { workspace = true, optional = true } +windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true anyhow.workspace = true diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index cd84b12d81..acf0948896 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -17,6 +17,7 @@ use tokio::task::JoinHandle; pub use windmill_common::db::DB; use windmill_common::{error::Error, utils::generate_lock_id}; +#[allow(unused_imports)] pub use windmill_api_auth::{ApiAuthed, OptJobAuthed}; async fn current_database(conn: &mut PgConnection) -> Result { diff --git a/backend/windmill-api/src/flow_conversations.rs b/backend/windmill-api/src/flow_conversations.rs index 4292139862..6f3b879ff7 100644 --- a/backend/windmill-api/src/flow_conversations.rs +++ b/backend/windmill-api/src/flow_conversations.rs @@ -128,24 +128,26 @@ async fn delete_conversation( tx.commit().await?; // Delete associated memory in background (non-blocking cleanup) - let w_id_clone = w_id.clone(); - let db_clone = db.clone(); - tokio::spawn(async move { - if let Err(e) = windmill_worker::memory_oss::delete_conversation_memory( - &db_clone, - &w_id_clone, - conversation_id, - ) - .await - { - tracing::error!( - "Failed to delete memory for conversation {} in workspace {}: {:?}", + { + let w_id_clone = w_id.clone(); + let db_clone = db.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_common::flow_conversations::delete_conversation_memory( + &db_clone, + &w_id_clone, conversation_id, - w_id_clone, - e - ); - } - }); + ) + .await + { + tracing::error!( + "Failed to delete memory for conversation {} in workspace {}: {:?}", + conversation_id, + w_id_clone, + e + ); + } + }); + } Ok(format!("Conversation {} deleted", conversation_id)) } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index fbfd64e2ef..5473a93d0d 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -49,10 +49,10 @@ use windmill_common::{ scripts::Schema, utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, }; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_queue::WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT; use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel}; -use windmill_worker::scoped_dependency_map::ScopedDependencyMap; pub fn workspaced_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index ecf86079bf..667204e7e9 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -29,17 +29,20 @@ use url::Url; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; +#[cfg(feature = "inline_preview")] use windmill_common::client::AuthedClient; use windmill_common::db::UserDbWithAuthed; use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; +#[cfg(feature = "inline_preview")] +use windmill_common::jobs::RunInlinePreviewScriptFnParams; use windmill_common::jobs::{ - format_completed_job_result, format_result, DynamicInput, RunInlinePreviewScriptFnParams, - ENTRYPOINT_OVERRIDE, + format_completed_job_result, format_result, DynamicInput, ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettings, }; +#[cfg(feature = "inline_preview")] use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams}; use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::scripts::ScriptRunnableSettingsInline; @@ -52,11 +55,14 @@ use windmill_common::workspace_dependencies::{ use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; +#[cfg(feature = "inline_preview")] use windmill_parser::asset_parser::AssetKind; +#[cfg(feature = "inline_preview")] use windmill_worker::get_worker_internal_server_inline_utils; use windmill_common::variables::get_workspace_key; +#[cfg(feature = "inline_preview")] use crate::db::OptJobAuthed; use crate::triggers::trigger_helpers::{FlowId, ScriptId}; use crate::{ @@ -2857,6 +2863,7 @@ struct Preview { format: Option, } +#[cfg(feature = "inline_preview")] #[derive(Debug, Deserialize)] struct PreviewInline { content: String, @@ -4591,6 +4598,7 @@ async fn run_preview_script( Ok((StatusCode::CREATED, uuid.to_string())) } +#[cfg(feature = "inline_preview")] async fn run_inline_preview_script( OptJobAuthed { authed, job_id }: OptJobAuthed, Tokened { token }: Tokened, @@ -4627,6 +4635,14 @@ async fn run_inline_preview_script( Ok(Json(to_raw_value(&result)).into_response()) } +#[cfg(not(feature = "inline_preview"))] +async fn run_inline_preview_script() -> error::Result { + Err(error::Error::InternalErr( + "inline preview requires the worker feature".to_string(), + )) +} + +#[cfg(feature = "inline_preview")] fn register_potential_assets_on_inline_execution( job_id: Uuid, w_id: &str, diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 5032bfe5db..7345c02e3c 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -38,7 +38,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; -use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap}; +use windmill_dep_map::process_relative_imports; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ assets::{ @@ -1146,7 +1147,6 @@ async fn create_script_internal<'c>( let content = ns.content.clone(); let language = ns.language.clone(); tokio::spawn(async move { - // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete tokio::time::sleep(std::time::Duration::from_secs(10)).await; if let Err(e) = process_relative_imports( &db2, diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs index 0212df6da6..2631e4fa37 100644 --- a/backend/windmill-api/src/secret_backend_ext.rs +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -17,10 +17,7 @@ #[cfg(all(feature = "private", feature = "enterprise"))] use std::sync::Arc; -use windmill_common::{ - db::DB, - error::Result, -}; +use windmill_common::{db::DB, error::Result}; #[cfg(all(feature = "private", feature = "enterprise"))] use windmill_common::error::Error; diff --git a/backend/windmill-api/src/webhook_util.rs b/backend/windmill-api/src/webhook_util.rs index ec9916a396..0287191efc 100644 --- a/backend/windmill-api/src/webhook_util.rs +++ b/backend/windmill-api/src/webhook_util.rs @@ -1,154 +1 @@ -use std::time::Duration; - -use quick_cache::sync::Cache; -use serde::Serialize; -use tokio::{select, sync::mpsc}; - -#[cfg(feature = "prometheus")] -use windmill_common::METRICS_ENABLED; - -use crate::db::DB; -use windmill_common::oauth2::InstanceEvent; -use windmill_common::utils::configure_client; - -#[cfg(feature = "prometheus")] -lazy_static::lazy_static! { - // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. - static ref WEBHOOK_REQUEST_COUNT: prometheus::Histogram = prometheus::register_histogram!( - "webhook_request", - "Histogram of webhook requests made" - ) - .unwrap(); - -} - -lazy_static::lazy_static! { - - pub static ref INSTANCE_EVENTS_WEBHOOK: Option = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(); - - pub static ref WEBHOOK_CACHE: Cache> = Cache::new(100); - -} - -pub enum WebhookPayload { - WorkspaceEvent(String, WebhookMessage), - InstanceEvent(InstanceEvent), -} - -#[derive(Serialize)] -#[serde(tag = "type")] -pub enum WebhookMessage { - // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, -} - -#[derive(Clone)] -pub struct WebhookShared { - pub channel: mpsc::UnboundedSender, -} - -impl WebhookShared { - pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { - let (tx, mut rx) = mpsc::unbounded_channel::(); - let _process = tokio::spawn(async move { - let client = configure_client( - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(5)) - // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5)), - ) - .build() - .unwrap(); - - loop { - select! { - biased; - _ = shutdown_rx.recv() => break, - r = rx.recv() => match r { - Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => { - let webhook_opt = match WEBHOOK_CACHE.get(&workspace_id) { - Some(guard) => { - guard - }, - None => { - let Ok(mut webhook_opt) = - sqlx::query_scalar!( - "SELECT webhook FROM workspace_settings WHERE workspace_id = $1", - workspace_id - ) - .fetch_one( - &db, - ) - .await else { - tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}"); - continue; - }; - if webhook_opt.as_ref().is_some_and(|x| x.is_empty()) { - webhook_opt = None; - } - WEBHOOK_CACHE.insert(workspace_id, webhook_opt.clone()); - webhook_opt - } - }; - if let Some(url) = webhook_opt { - #[cfg(feature = "prometheus")] - let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; - tracing::info!("Sending webhook message to {}", url); - let _ = client.post(url).json(&message).send().await; - #[cfg(feature = "prometheus")] - timer.map(|x| x.stop_and_record()); - } - }, - Some(WebhookPayload::InstanceEvent(event)) => { - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; - let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await; - if let Err(e) = r { - tracing::error!("Error sending instance event: {}", e); - } - }, - None => break, - }, - } - } - }); - - Self { channel: tx } - } - - pub fn send_message(&self, workspace_id: String, message: WebhookMessage) { - let _ = self.channel.send(WebhookPayload::WorkspaceEvent( - workspace_id.clone(), - message, - )); - } - - pub fn send_instance_event(&self, event: InstanceEvent) { - if INSTANCE_EVENTS_WEBHOOK.is_none() { - return; - } - let _ = self.channel.send(WebhookPayload::InstanceEvent(event)); - } -} +pub use windmill_common::webhook::*; diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index 2330de200a..d887923f5b 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -13,7 +13,7 @@ use windmill_common::{ workspace_dependencies::WorkspaceDependencies, DB, }; -use windmill_worker::workspace_dependencies::{ +use windmill_dep_map::workspace_dependencies::{ trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 93c524f5fc..f5e1f1fbe7 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -54,10 +54,10 @@ use windmill_common::{ oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, utils::{paginate, rd_string, require_admin, Pagination}, }; -use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; -use windmill_worker::scoped_dependency_map::{ +use windmill_dep_map::scoped_dependency_map::{ DependencyDependent, DependencyMap, ScopedDependencyMap, }; +use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; #[cfg(feature = "enterprise")] use windmill_common::utils::require_admin_or_devops; diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 4ba43e86a6..33056f59fa 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use sqlx::{self, FromRow}; use uuid::Uuid; +use crate::db::DB; use crate::error::Result; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, sqlx::Type)] @@ -128,3 +129,20 @@ pub async fn add_message_to_conversation_tx( Ok(()) } + +/// Delete all memory for a conversation from the database +pub async fn delete_conversation_memory( + db: &DB, + workspace_id: &str, + conversation_id: Uuid, +) -> Result<()> { + sqlx::query!( + "DELETE FROM ai_agent_memory WHERE workspace_id = $1 AND conversation_id = $2", + workspace_id, + conversation_id + ) + .execute(db) + .await?; + + Ok(()) +} diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml new file mode 100644 index 0000000000..ae81f5d1a7 --- /dev/null +++ b/backend/windmill-dep-map/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "windmill-dep-map" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_dep_map" +path = "src/lib.rs" + +[features] +default = [] +python = ["dep:windmill-parser-py-imports"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-py-imports = { workspace = true, optional = true } +sqlx.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +lazy_static.workspace = true +chrono.workspace = true +itertools.workspace = true +uuid.workspace = true diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs new file mode 100644 index 0000000000..43aceb1d38 --- /dev/null +++ b/backend/windmill-dep-map/src/lib.rs @@ -0,0 +1,203 @@ +pub mod scoped_dependency_map; +pub mod trigger_dependents; +pub mod workspace_dependencies; + +use std::collections::HashMap; +use std::path::{Component, Path, PathBuf}; + +use serde_json::value::RawValue; +use sqlx::types::Json; +use uuid::Uuid; +use windmill_common::error; +use windmill_common::scripts::ScriptLang; +use windmill_common::utils::WarnAfterExt; +use windmill_common::workspace_dependencies::{ + WorkspaceDependencies, WorkspaceDependenciesPrefetched, +}; +use windmill_parser_ts::parse_expr_for_imports; + +fn try_normalize(path: &Path) -> Option { + let mut ret = PathBuf::new(); + + for component in path.components() { + match component { + Component::Prefix(..) | Component::RootDir => return None, + Component::CurDir => {} + Component::ParentDir => { + if !ret.pop() { + return None; + } + } + Component::Normal(c) => { + ret.push(c); + } + } + } + + Some(ret) +} + +fn parse_ts_relative_imports( + raw_code: &str, + script_path: &str, +) -> windmill_common::error::Result> { + let mut relative_imports = vec![]; + let r = parse_expr_for_imports(raw_code, true)?; + for import in r { + let import = import.trim_end_matches(".ts"); + if import.starts_with("/") { + relative_imports.push(import.trim_start_matches("/").to_string()); + } else if import.starts_with(".") { + let normalized = try_normalize(std::path::Path::new(&format!( + "{}/../{}", + script_path, import + ))); + if let Some(normalized) = normalized { + let normalized = normalized.to_str().unwrap().to_string(); + relative_imports.push(normalized); + } else { + tracing::error!("error canonicalizing path: {script_path} with import {import}"); + } + } + } + + Ok(relative_imports) +} + +pub fn extract_relative_imports( + raw_code: &str, + script_path: &str, + language: &Option, +) -> Option> { + match language { + #[cfg(feature = "python")] + Some(ScriptLang::Python3) => { + windmill_parser_py_imports::parse_relative_imports(&raw_code, script_path).ok() + } + Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) | Some(ScriptLang::Deno) => { + parse_ts_relative_imports(&raw_code, script_path).ok() + } + _ => None, + } +} + +pub fn extract_referenced_paths( + raw_code: &str, + script_path: &str, + language: Option, +) -> Option> { + let mut referenced_paths = vec![]; + if let Some(wk_deps_refs) = language + .and_then(|l| l.extract_workspace_dependencies_annotated_refs(raw_code, script_path)) + .map(|r| r.external) + { + let l = language.expect("should be some"); + for wk_deps_ref in wk_deps_refs { + if let Some(path) = WorkspaceDependencies::to_path(&Some(wk_deps_ref), l).ok() { + referenced_paths.push(path); + }; + } + } else if let (Some(l), true /* Only if it is not blacklisted */) = ( + language, + WorkspaceDependenciesPrefetched::is_external_references_permitted(script_path), + ) { + // we assume all runnables without annotated dependencies reference default dependencies file. + WorkspaceDependencies::to_path(&None, l) + .ok() + .inspect(|p| referenced_paths.push(p.to_owned())); + } + + if let Some(relative_imports) = extract_relative_imports(raw_code, script_path, &language) { + referenced_paths.extend(relative_imports); + } + + if referenced_paths.is_empty() { + None + } else { + Some(referenced_paths) + } +} + +pub async fn process_relative_imports( + db: &sqlx::Pool, + _job_id: Option, + args: Option<&Json>>>, + w_id: &str, + script_path: &str, + parent_path: Option, + deployment_message: Option, + code: &str, + script_lang: &Option, + permissioned_as_email: &str, + created_by: &str, + permissioned_as: &str, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + use trigger_dependents::trigger_dependents_to_recompute_dependencies; + + // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled + { + let mut tx = db.begin().await?; + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &w_id, + script_path, + "script", + &parent_path, + db, + ) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(&code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + } + + { + let mut already_visited = args + .map(|x| { + x.get("already_visited") + .map(|v| serde_json::from_str::>(v.get()).ok()) + .flatten() + }) + .flatten() + .unwrap_or_default(); + + let importers = ScopedDependencyMap::get_dependents(script_path, w_id, db).await?; + + already_visited.push(script_path.to_string()); + match tokio::time::timeout( + core::time::Duration::from_secs(60), + Box::pin(trigger_dependents_to_recompute_dependencies( + w_id, + importers, + deployment_message, + parent_path, + permissioned_as_email, + created_by, + permissioned_as, + db, + already_visited, + )), + ) + .warn_after_seconds(10) + .await + { + Ok(Err(e)) => { + tracing::error!(%e, "error triggering dependents to recompute dependencies") + } + Err(e) => { + tracing::error!(%e, "triggering dependents to recompute dependencies has timed out") + } + _ => {} + } + } + + Ok(()) +} diff --git a/backend/windmill-worker/src/scoped_dependency_map.rs b/backend/windmill-dep-map/src/scoped_dependency_map.rs similarity index 86% rename from backend/windmill-worker/src/scoped_dependency_map.rs rename to backend/windmill-dep-map/src/scoped_dependency_map.rs index 9e9884e538..49646524f0 100644 --- a/backend/windmill-worker/src/scoped_dependency_map.rs +++ b/backend/windmill-dep-map/src/scoped_dependency_map.rs @@ -11,14 +11,10 @@ use windmill_common::{ use std::collections::HashSet; -use crate::worker_lockfiles::extract_referenced_paths; - -// TODO: To be removed in future versions lazy_static::lazy_static! { pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok(); } -// TODO: Rename to DependencyRelation #[derive(Serialize)] pub struct DependencyMap { pub workspace_id: Option, @@ -48,7 +44,7 @@ impl ScopedDependencyMap { /// Calls DB, however is assumed to be called once per dependency job /// AND is scoped to smaller subset of data /// So it is not too expensive - pub(crate) async fn fetch_maybe_rearranged<'a>( + pub async fn fetch_maybe_rearranged<'a>( w_id: &str, importer_path: &str, importer_kind: &str, @@ -123,7 +119,7 @@ SELECT importer_node_id, imported_path /// Add missing entries to `dependency_map` /// Remove matching entries - pub(crate) async fn patch<'c>( + pub async fn patch<'c>( &mut self, referenced_paths: Option>, node_id: String, // Flow Step/Node ID @@ -134,7 +130,7 @@ SELECT importer_node_id, imported_path Ok(tx) } - pub(crate) async fn patch_tx_ref<'c>( + pub async fn patch_tx_ref<'c>( &mut self, // NOTE: Referenced_paths should include all of the paths. referenced_paths: Option>, @@ -150,26 +146,12 @@ SELECT importer_node_id, imported_path return Ok(()); }; - // This does: - // 1. remove all relative imports from relative_imports that ARE tracked in dependency_map - // 2. remove corresponding trackers from dependency_map - // - // After this operation `relative_imports` variable has only untracked imports. - // We will handle those in the next expression. - // - // After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports - // these are going to be clean up by calling [dissolve] - // NOTE: `retain` iterates over vec and remove the ones whose closures returned false. referenced_paths.retain(|imported_path| { !self .to_delete - // As dmap is HashSet, removing is O(1) operation - // thus making entire process very efficient - // NOTE: `remove` returns true if item was removed and false if wasn't. .remove(&(node_id.to_owned(), imported_path.to_owned())) }); - // As mentioned above, usually this will always be empty. if !referenced_paths.is_empty() { tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}", &node_id, @@ -197,7 +179,7 @@ SELECT importer_node_id, imported_path } /// clean orphan entries from `dependency_map` - pub(crate) async fn dissolve<'a>( + pub async fn dissolve<'a>( self, mut tx: sqlx::Transaction<'a, sqlx::Postgres>, ) -> sqlx::Transaction<'a, sqlx::Postgres> { @@ -210,7 +192,6 @@ SELECT importer_node_id, imported_path tracing::info!("dissolving dependency_map: {:?}", &self); - // We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability. for (importer_node_id, imported_path) in self.to_delete.into_iter() { tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}", &self.importer_kind, @@ -218,7 +199,6 @@ SELECT importer_node_id, imported_path &importer_node_id, ); - // Dissolve MUST succeed. Error in dissolve MUST not block the execution. if let Err(err) = sqlx::query!( " DELETE FROM dependency_map @@ -265,7 +245,6 @@ SELECT importer_node_id, imported_path "discovered orphan entry in `dependency_map`. It will be healed automatically, however please report this issue to Windmill Team. It is also advised to rebuild maps in workspace settings in troubleshooting.", ); - // MUST succeed. Error MUST not block the execution. if let Err(err) = sqlx::query!( "DELETE FROM dependency_map WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND @@ -286,7 +265,7 @@ SELECT importer_node_id, imported_path tx } - pub(crate) async fn rebuild_map_unchecked<'c>( + pub async fn rebuild_map_unchecked( w_id: &str, db: &sqlx::Pool, ) -> Result { @@ -305,7 +284,7 @@ SELECT importer_node_id, imported_path tx = dmap .patch( - extract_referenced_paths(&sd.code, &r.path, smd.language), + crate::extract_referenced_paths(&sd.code, &r.path, smd.language), "".into(), tx, ) @@ -318,18 +297,13 @@ SELECT importer_node_id, imported_path } // Fetch only top level versions and paths - // It is not fetching value tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows"); for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? { if let Some(version) = r.version { - // To reduce stress on db try to fetch from cache - // Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date. let flow_data = cache::flow::fetch_version(&db.clone().into(), version).await?; - // Create map for specific flow let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "flow", db).await?; - // Traverse retrieved flow modules let mut tx = db.begin().await?; let mut to_process = vec![]; let mut modules_to_check = flow_data.flow.modules.iter().collect::>(); @@ -342,10 +316,9 @@ SELECT importer_node_id, imported_path FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| { match fmv { - // Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript. FlowModuleValue::RawScript { content, language, .. } => { to_process.push(( - extract_referenced_paths( + crate::extract_referenced_paths( content, &(r.path.clone() + "/flow"), Some(*language), @@ -353,9 +326,7 @@ SELECT importer_node_id, imported_path id.clone(), )); } - // But just in case we will also handle other cases. FlowModuleValue::FlowScript { .. } => { - // Abort will cancel transaction. return Err(Error::internal_err("FlowScript is not supposed to be in flow.")); } _ => {} @@ -382,7 +353,6 @@ SELECT importer_node_id, imported_path tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps"); for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? { if let Some(version) = r.version { - // TODO: Use cache when implemented. let value = sqlx::query_scalar!( "SELECT value FROM app_version WHERE id = $1 LIMIT 1", version @@ -395,7 +365,7 @@ SELECT importer_node_id, imported_path let mut to_process = vec![]; traverse_app_inline_scripts(&value, None, &mut |ais, id| { to_process.push(( - extract_referenced_paths( + crate::extract_referenced_paths( &ais.content, &(r.path.clone() + "/app"), ais.language, @@ -423,6 +393,7 @@ SELECT importer_node_id, imported_path Ok("Success".into()) } + /// Run if you want to rebuild maps on specific workspace. /// Potentially takes much time pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool) -> Result { @@ -459,11 +430,11 @@ SELECT importer_node_id, imported_path sqlx::query_as!( DependencyDependent, r#" - SELECT + SELECT importer_path, - importer_kind::text as "importer_kind!", -- sqlx thinks this is nullable somehow, so enfore with ! + importer_kind::text as "importer_kind!", array_agg(importer_node_id) as importer_node_ids - FROM dependency_map + FROM dependency_map WHERE workspace_id = $1 AND imported_path = $2 GROUP BY importer_path, importer_kind "#, diff --git a/backend/windmill-dep-map/src/trigger_dependents.rs b/backend/windmill-dep-map/src/trigger_dependents.rs new file mode 100644 index 0000000000..d2e3fd56dd --- /dev/null +++ b/backend/windmill-dep-map/src/trigger_dependents.rs @@ -0,0 +1,220 @@ +use std::collections::HashMap; + +use chrono::{Duration, Utc}; +use itertools::Itertools; +use serde_json::value::RawValue; +use windmill_common::error; +use windmill_common::jobs::JobPayload; +use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::scripts::ScriptHash; +use windmill_common::worker::to_raw_value; +use windmill_queue::PushIsolationLevel; + +use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; + +lazy_static::lazy_static! { + static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( + if cfg!(test) { 15 } else { 5 } + ); +} + +pub async fn trigger_dependents_to_recompute_dependencies( + w_id: &str, + importers: Vec, + deployment_message: Option, + parent_path: Option, + email: &str, + created_by: &str, + permissioned_as: &str, + db: &sqlx::Pool, + already_visited: Vec, +) -> error::Result<()> { + tracing::debug!( + "Triggering dependents to recompute dependencies: {}", + importers.iter().map(|dd| &dd.importer_path).join(",") + ); + for DependencyDependent { importer_path, importer_kind, importer_node_ids } in importers.iter() + { + tracing::trace!("Processing dependency: {:?}", importer_path); + if already_visited.contains(importer_path) { + tracing::trace!("Skipping already visited dependency"); + continue; + } + + let mut tx = db.clone().begin().await?; + let mut args: HashMap> = HashMap::new(); + if let Some(ref dm) = deployment_message { + args.insert("deployment_message".to_string(), to_raw_value(&dm)); + } + if let Some(ref p_path) = parent_path { + args.insert("common_dependency_path".to_string(), to_raw_value(&p_path)); + } + + args.insert( + "already_visited".to_string(), + to_raw_value(&already_visited), + ); + + args.insert( + "triggered_by_relative_import".to_string(), + to_raw_value(&true), + ); + + let mut debouncing_settings = DebouncingSettings { + debounce_key: Some(format!("{w_id}:{importer_path}:dependency")), + debounce_delay_s: Some(5), + ..Default::default() + }; + + let job_payload = match importer_kind.as_str() { + "script" => match sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(hash) => { + tracing::debug!("newest hash for {} is: {hash}", importer_path); + + let info = + windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; + + JobPayload::Dependencies { + path: importer_path.clone(), + hash: ScriptHash(hash), + language: info.language, + dedicated_worker: info.dedicated_worker, + debouncing_settings, + } + } + None => { + ScopedDependencyMap::clear_map_for_item( + importer_path, + w_id, + "script", + tx, + &None, + ) + .await + .commit() + .await?; + continue; + } + }, + + "flow" => match sqlx::query_scalar!( + "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + tracing::debug!("Handling flow dependency update for: {}", importer_path); + + args.insert( + "nodes_to_relock".to_string(), + to_raw_value(&importer_node_ids), + ); + + debouncing_settings.debounce_args_to_accumulate = Some(vec!["nodes_to_relock".into()]); + + JobPayload::FlowDependencies { + path: importer_path.clone(), + version, + dedicated_worker: None, + debouncing_settings, + } + } + None => { + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "flow", tx, &None) + .await + .commit() + .await?; + continue; + } + }, + + "app" => match sqlx::query_scalar!( + "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + tracing::debug!("Handling app dependency update for: {}", importer_path); + + args.insert( + "components_to_relock".to_string(), + to_raw_value(importer_node_ids), + ); + + debouncing_settings.debounce_args_to_accumulate = Some(vec!["components_to_relock".into()]); + + JobPayload::AppDependencies { path: importer_path.clone(), version, debouncing_settings } + } + None => { + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "app", tx, &None) + .await + .commit() + .await?; + continue; + } + }, + + _ => { + tracing::error!( + "unexpected importer kind: {kind:?} for path {path}", + kind = importer_kind, + path = importer_path + ); + continue; + } + }; + + tracing::debug!("Pushing dependency job for: {}", importer_path); + let (job_uuid, new_tx) = windmill_queue::push( + db, + PushIsolationLevel::Transaction(tx), + &w_id, + job_payload, + windmill_queue::PushArgs { args: &args, extra: None }, + &created_by, + email, + permissioned_as.to_string(), + Some("trigger.dependents.to.recompute.dependencies"), + Some(Utc::now() + Duration::seconds(*DEPENDENCY_JOB_DEBOUNCE_DELAY as i64)), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await?; + + tracing::info!( + "pushed dependency job due to common python path: {job_uuid} for path {path}", + path = importer_path, + ); + new_tx.commit().await?; + } + Ok(()) +} diff --git a/backend/windmill-worker/src/workspace_dependencies.rs b/backend/windmill-dep-map/src/workspace_dependencies.rs similarity index 54% rename from backend/windmill-worker/src/workspace_dependencies.rs rename to backend/windmill-dep-map/src/workspace_dependencies.rs index 281b08a32c..6186556a25 100644 --- a/backend/windmill-worker/src/workspace_dependencies.rs +++ b/backend/windmill-dep-map/src/workspace_dependencies.rs @@ -5,7 +5,8 @@ use windmill_common::{ }; use crate::{ - scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies, + scoped_dependency_map::ScopedDependencyMap, + trigger_dependents::trigger_dependents_to_recompute_dependencies, }; #[derive(sqlx::FromRow, Clone, Serialize, Deserialize, Hash, Debug)] @@ -16,7 +17,6 @@ pub struct NewWorkspaceDependencies { /// If None, will use description of previous version /// If there is no older versions, will set to default pub description: Option, - // TODO: Make Option, or optimize it in any other way. pub content: String, } @@ -31,16 +31,12 @@ impl NewWorkspaceDependencies { metadata: (String, String, String), db: sqlx::Pool, ) -> error::Result { - // Check if all workers support workspace dependencies feature windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() .await?; let path = WorkspaceDependencies::to_path(&self.name, self.language)?; - // If it is unnamed then we want to rebuild dependency map. Otherwise trigger dependents to recompute locks will not work - // NOTE: We rebuild first, even before creating new w deps. We want to make sure that if rebuild failed, then no new default workspace dependencies were created. if self.name.is_none() { - // Check if we already rebuilt the map for this workspace by checking if the setting exists let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id); let already_rebuilt = windmill_common::global_settings::load_value_from_global_settings( @@ -57,7 +53,6 @@ impl NewWorkspaceDependencies { ); ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, &db).await?; - // Mark as rebuilt by creating the setting windmill_common::global_settings::set_value_in_global_settings( &db, &setting_name, @@ -80,7 +75,7 @@ impl NewWorkspaceDependencies { let prev_description = sqlx::query_scalar!( " UPDATE workspace_dependencies - SET archived = true + SET archived = true WHERE archived = false AND name IS NOT DISTINCT FROM $1 AND workspace_id = $2 @@ -97,7 +92,7 @@ impl NewWorkspaceDependencies { let new_id = sqlx::query_scalar!( " INSERT INTO workspace_dependencies(name, workspace_id, content, language, description) - VALUES ($1, $2, $3, $4, $5) + VALUES ($1, $2, $3, $4, $5) RETURNING id ", self.name.clone(), @@ -141,17 +136,12 @@ pub async fn trigger_dependents_to_recompute_dependencies_in_the_background( language = ?language, "waiting for cache timeout after creating first unnamed workspace dependencies" ); - // Wait for cache timeout. - // For context, workers have cache on whether the unnamed workspace dependencies exists or not. - // when we trigger dependents to recompoute dependencies we want to make sure all workers are having cache timed out. - // otherwise it would result into bug, when workers skip fetch of workspace dependencies because they think they don't exist. tokio::time::sleep(EXISTS_CACHE_TIMEOUT).await; } - // It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies if let Err(e) = trigger_dependents_to_recompute_dependencies( &workspace_id, - match crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( + match ScopedDependencyMap::get_dependents( path.as_str(), &workspace_id, &db, @@ -189,114 +179,5 @@ pub async fn trigger_dependents_to_recompute_dependencies_in_the_background( }); } -// Type aliases for backward compatibility pub type RawRequirements = WorkspaceDependencies; pub type NewRawRequirements = NewWorkspaceDependencies; - -#[cfg(test)] -mod workspace_dependencies_tests { - - // // TODO: test all cases when it should reject. - // #[cfg(feature = "python")] - // mod new_workspace_dependencies { - // use windmill_common::scripts::ScriptLang; - - // use crate::workspace_dependencies::NewWorkspaceDependencies; - - // #[sqlx::test( - // fixtures("../../tests/fixtures/base.sql",), - // migrations = "../migrations" - // )] - // async fn test_create(db: sqlx::Pool) -> anyhow::Result<()> { - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // name: None, - // description: None, - // content: "global:rev1".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // 1 - // ); - - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // name: Some("rrs1".to_owned()), - // description: None, - // content: "rrs1:rev1".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // 2 - // ); - - // assert!(NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::DuckDb, - // description: None, - // name: None, - // content: "".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .is_err()); - - // // Will act as redeployment - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // description: None, - // name: Some("rrs1".to_owned()), - // content: "rrs1:rev2".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // // It will just increment id - // 3 - // ); - // Ok(()) - // } - - // #[sqlx::test( - // fixtures("../../tests/fixtures/base.sql",), - // migrations = "../migrations" - // )] - // async fn violate_constraints(db: sqlx::Pool) -> anyhow::Result<()> { - // let db = &db; - // let create = |name| { - // sqlx::query_scalar!( - // " - // INSERT INTO workspace_dependencies(name, workspace_id, content, language) - // VALUES ($1, 'test-workspace', 'test', 'python3') - // RETURNING id - // ", - // name - // ) - // .fetch_one(db) - // }; - - // assert_eq!(create(Some("test".to_owned())).await.unwrap(), 1); - // assert_eq!(create(None).await.unwrap(), 2); - - // assert!(create(Some("test".to_owned())).await.is_err()); - // assert!(create(None).await.is_err()); - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM workspace_dependencies",) - // .fetch_one(db) - // .await - // .unwrap() - // .unwrap(), - // 2 - // ); - // Ok(()) - // } - // } -} diff --git a/backend/windmill-trigger/Cargo.toml b/backend/windmill-trigger/Cargo.toml index c35a6ffeaa..3cff7db14e 100644 --- a/backend/windmill-trigger/Cargo.toml +++ b/backend/windmill-trigger/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -enterprise = ["windmill-common/enterprise"] +enterprise = ["windmill-common/enterprise", "windmill-api-jobs/enterprise"] cloud = ["windmill-common/cloud"] [dependencies] diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 76d1fbbdcf..51df8ca7f9 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -32,7 +32,7 @@ dind = ["dep:bollard"] php = ["dep:windmill-parser-php"] mysql = ["dep:mysql_async"] oracledb = ["dep:oracle"] -python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"] +python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports", "windmill-dep-map/python"] csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] @@ -44,6 +44,7 @@ bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"] [dependencies] windmill-queue.workspace = true +windmill-dep-map.workspace = true windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. windmill-common = { workspace = true, default-features = false } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index 726a8e27e2..8eb3609816 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -500,7 +500,11 @@ impl QueryBuilder for AnthropicQueryBuilder { // For Vertex AI, the model is specified in the URL path // Expected base_url format: https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{location}/publishers/anthropic/models // We append the model and :streamRawPredict - format!("{}/{}:streamRawPredict", base_url.trim_end_matches('/'), model) + format!( + "{}/{}:streamRawPredict", + base_url.trim_end_matches('/'), + model + ) } else { format!("{}/messages", base_url) } diff --git a/backend/windmill-worker/src/ai/providers/bedrock.rs b/backend/windmill-worker/src/ai/providers/bedrock.rs index a98fed44af..a86bf3250c 100644 --- a/backend/windmill-worker/src/ai/providers/bedrock.rs +++ b/backend/windmill-worker/src/ai/providers/bedrock.rs @@ -17,13 +17,13 @@ use std::collections::HashMap; use windmill_common::{client::AuthedClient, error::Error}; // Re-export from shared module for use by other parts of the worker -pub use windmill_common::ai_bedrock::{check_env_credentials, BedrockClient}; use windmill_common::ai_bedrock::{ bedrock_stream_event_is_block_stop, bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_start, build_tool_config, create_inference_config, format_bedrock_error, openai_messages_to_bedrock, streaming_tool_calls_to_openai, StreamingToolCall, }; +pub use windmill_common::ai_bedrock::{check_env_credentials, BedrockClient}; // ============================================================================ // Query Builder (Worker-specific orchestration) @@ -161,9 +161,7 @@ impl BedrockQueryBuilder { if let Some(processor) = stream_event_processor.as_ref() { processor .send( - StreamingEvent::TokenDelta { - content: text_delta, - }, + StreamingEvent::TokenDelta { content: text_delta }, &mut events_str, ) .await?; @@ -185,9 +183,8 @@ impl BedrockQueryBuilder { } // Extract usage from Metadata event - if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata( - metadata, - ) = &event + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = + &event { if let Some(token_usage) = metadata.usage() { usage = Some( diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index 7f841ea07f..51bb6e3ec3 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -473,8 +473,9 @@ impl QueryBuilder for OpenAIQueryBuilder { parser.parse_events(response).await?; // Convert OpenAI Responses usage to TokenUsage - let usage = - parser.usage.map(|u| TokenUsage::new(u.input_tokens, u.output_tokens, u.total_tokens)); + let usage = parser + .usage + .map(|u| TokenUsage::new(u.input_tokens, u.output_tokens, u.total_tokens)); Ok(ParsedResponse::Text { content: if parser.accumulated_content.is_empty() { @@ -518,9 +519,7 @@ impl QueryBuilder for OpenAIQueryBuilder { "image_generation_call" => { if output.status.as_deref() == Some("completed") { if let Some(ref base64_image) = output.result { - return Ok(ParsedResponse::Image { - base64_data: base64_image.clone(), - }); + return Ok(ParsedResponse::Image { base64_data: base64_image.clone() }); } } } diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs index e15ac94ec7..62cee60420 100644 --- a/backend/windmill-worker/src/ai/providers/openrouter.rs +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -117,9 +117,7 @@ impl QueryBuilder for OpenRouterQueryBuilder { .and_then(|images| images.first()) { if let Some(base64_data) = image.image_url.url.strip_prefix("data:image/png;base64,") { - return Ok(ParsedResponse::Image { - base64_data: base64_data.to_string(), - }); + return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() }); } } diff --git a/backend/windmill-worker/src/ai/providers/other.rs b/backend/windmill-worker/src/ai/providers/other.rs index e000cd11b7..a92caca923 100644 --- a/backend/windmill-worker/src/ai/providers/other.rs +++ b/backend/windmill-worker/src/ai/providers/other.rs @@ -225,8 +225,8 @@ impl QueryBuilder for OtherQueryBuilder { } // Convert OpenAI Chat Completions usage to TokenUsage - let usage = - openai_usage.map(|u| TokenUsage::new(u.prompt_tokens, u.completion_tokens, u.total_tokens)); + let usage = openai_usage + .map(|u| TokenUsage::new(u.prompt_tokens, u.completion_tokens, u.total_tokens)); Ok(ParsedResponse::Text { content: if accumulated_content.is_empty() { diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 2bbf93c69a..10e510a492 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -7,10 +7,8 @@ use windmill_queue::MiniPulledJob; use crate::{ ai::{ providers::{ - anthropic::AnthropicQueryBuilder, - google_ai::GoogleAIQueryBuilder, - openai::{OpenAIQueryBuilder}, - openrouter::OpenRouterQueryBuilder, + anthropic::AnthropicQueryBuilder, google_ai::GoogleAIQueryBuilder, + openai::OpenAIQueryBuilder, openrouter::OpenRouterQueryBuilder, other::OtherQueryBuilder, }, types::*, diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 34a2d789ec..5db641b80d 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -1,4 +1,3 @@ -use windmill_common::ai_types::OpenAIToolCall; use crate::ai::query_builder::StreamEventProcessor; use crate::ai::types::McpToolSource; use crate::ai::types::*; @@ -21,6 +20,7 @@ use mappable_rc::Marc; use serde_json::value::RawValue; use std::{collections::HashMap, sync::Arc}; use uuid::Uuid; +use windmill_common::ai_types::OpenAIToolCall; use windmill_common::flows::InputTransform; use windmill_common::jobs::JobPayload; diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index 0955e7c7a1..11e2243bf9 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -174,10 +174,18 @@ pub struct ProviderResource { #[serde(default, deserialize_with = "empty_string_as_none")] pub region: Option, #[allow(dead_code)] - #[serde(alias = "awsAccessKeyId", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsAccessKeyId", + default, + deserialize_with = "empty_string_as_none" + )] pub aws_access_key_id: Option, #[allow(dead_code)] - #[serde(alias = "awsSecretAccessKey", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsSecretAccessKey", + default, + deserialize_with = "empty_string_as_none" + )] pub aws_secret_access_key: Option, /// Platform for Anthropic API (standard or google_vertex_ai) #[serde(default)] @@ -202,10 +210,7 @@ impl ProviderWithResource { pub async fn get_base_url(&self, db: &DB) -> Result { self.kind - .get_base_url( - self.resource.base_url.clone(), - db, - ) + .get_base_url(self.resource.base_url.clone(), db) .await } @@ -295,8 +300,10 @@ impl TokenUsage { self.total_tokens = add_option(self.total_tokens, other.total_tokens); self.cache_read_input_tokens = add_option(self.cache_read_input_tokens, other.cache_read_input_tokens); - self.cache_write_input_tokens = - add_option(self.cache_write_input_tokens, other.cache_write_input_tokens); + self.cache_write_input_tokens = add_option( + self.cache_write_input_tokens, + other.cache_write_input_tokens, + ); } } @@ -704,7 +711,9 @@ impl OpenAPISchema { if let Some(ref other_definitions) = other.definitions { let definitions = self.definitions.get_or_insert_with(HashMap::new); for (key, value) in other_definitions { - definitions.entry(key.clone()).or_insert_with(|| value.clone()); + definitions + .entry(key.clone()) + .or_insert_with(|| value.clone()); } } @@ -861,7 +870,10 @@ mod tests { schema.make_strict(); assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Expected additionalProperties to be false" ); } @@ -874,33 +886,36 @@ mod tests { schema.make_strict(); assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(true))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(true)) + ), "Expected additionalProperties to remain true (user-specified)" ); } #[test] fn test_make_strict_all_properties_required() { - let mut schema = object_schema(vec![ - ("name", string_schema()), - ("age", integer_schema()), - ]); + let mut schema = object_schema(vec![("name", string_schema()), ("age", integer_schema())]); schema.required = Some(vec!["name".to_string()]); // Only name is required initially schema.make_strict(); let required = schema.required.as_ref().expect("required should be set"); - assert!(required.contains(&"name".to_string()), "name should be required"); - assert!(required.contains(&"age".to_string()), "age should be required"); + assert!( + required.contains(&"name".to_string()), + "name should be required" + ); + assert!( + required.contains(&"age".to_string()), + "age should be required" + ); assert_eq!(required.len(), 2, "Should have exactly 2 required fields"); } #[test] fn test_make_strict_non_required_becomes_nullable() { - let mut schema = object_schema(vec![ - ("name", string_schema()), - ("age", integer_schema()), - ]); + let mut schema = object_schema(vec![("name", string_schema()), ("age", integer_schema())]); schema.required = Some(vec!["name".to_string()]); // Only name is required schema.make_strict(); @@ -915,7 +930,10 @@ mod tests { match &age_prop.r#type { Some(SchemaType::Multiple(types)) => { - assert!(types.contains(&"integer".to_string()), "Should contain integer"); + assert!( + types.contains(&"integer".to_string()), + "Should contain integer" + ); assert!(types.contains(&"null".to_string()), "Should contain null"); } _ => panic!("Expected age to have multiple types including null"), @@ -953,12 +971,18 @@ mod tests { .expect("nested property should exist"); assert!( - matches!(nested_prop.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + nested_prop.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Nested object should have additionalProperties: false" ); // Nested object should have all properties required - let nested_required = nested_prop.required.as_ref().expect("nested required should be set"); + let nested_required = nested_prop + .required + .as_ref() + .expect("nested required should be set"); assert!(nested_required.contains(&"field".to_string())); } @@ -975,7 +999,10 @@ mod tests { let items = schema.items.as_ref().expect("items should exist"); assert!( - matches!(items.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + items.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Array items should have additionalProperties: false" ); } @@ -994,7 +1021,10 @@ mod tests { for (i, variant) in schema.one_of.as_ref().unwrap().iter().enumerate() { assert!( - matches!(variant.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + variant.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "oneOf variant {} should have additionalProperties: false", i ); @@ -1015,7 +1045,10 @@ mod tests { for (i, variant) in schema.any_of.as_ref().unwrap().iter().enumerate() { assert!( - matches!(variant.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + variant.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "anyOf variant {} should have additionalProperties: false", i ); @@ -1028,10 +1061,7 @@ mod tests { let mut defs = HashMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); - let mut schema = OpenAPISchema { - defs: Some(defs), - ..Default::default() - }; + let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; schema.make_strict(); @@ -1043,7 +1073,10 @@ mod tests { .expect("MyType def should exist"); assert!( - matches!(my_type.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + my_type.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "$defs schema should have additionalProperties: false" ); } @@ -1054,10 +1087,7 @@ mod tests { let mut definitions = HashMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); - let mut schema = OpenAPISchema { - definitions: Some(definitions), - ..Default::default() - }; + let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; schema.make_strict(); @@ -1069,7 +1099,10 @@ mod tests { .expect("MyType definition should exist"); assert!( - matches!(my_type.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + my_type.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "definitions schema should have additionalProperties: false" ); } @@ -1168,7 +1201,10 @@ mod tests { schema.make_strict(); // allOf should be removed - assert!(schema.all_of.is_none(), "allOf should be removed after flattening"); + assert!( + schema.all_of.is_none(), + "allOf should be removed after flattening" + ); // Properties should be merged let props = schema.properties.as_ref().expect("properties should exist"); @@ -1183,7 +1219,10 @@ mod tests { // Should have additionalProperties: false (from make_strict) assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Should have additionalProperties: false" ); } @@ -1205,8 +1244,14 @@ mod tests { // Both name and age should be in required (from merge + make_strict makes all required) let required = schema.required.as_ref().expect("required should be set"); - assert!(required.contains(&"name".to_string()), "name should be required"); - assert!(required.contains(&"age".to_string()), "age should be required"); + assert!( + required.contains(&"name".to_string()), + "name should be required" + ); + assert!( + required.contains(&"age".to_string()), + "age should be required" + ); } #[test] @@ -1265,7 +1310,7 @@ mod tests { let schema2 = OpenAPISchema { r#type: Some(SchemaType::Single("integer".to_string())), - minimum: Some(5.0), // More restrictive + minimum: Some(5.0), // More restrictive maximum: Some(100.0), ..Default::default() }; @@ -1278,8 +1323,16 @@ mod tests { schema.flatten_all_of(); // Should take the more restrictive minimum (5.0) - assert_eq!(schema.minimum, Some(5.0), "Should have more restrictive minimum"); - assert_eq!(schema.maximum, Some(100.0), "Should have maximum from schema2"); + assert_eq!( + schema.minimum, + Some(5.0), + "Should have more restrictive minimum" + ); + assert_eq!( + schema.maximum, + Some(100.0), + "Should have maximum from schema2" + ); } #[test] @@ -1288,15 +1341,10 @@ mod tests { let mut defs = HashMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); - let schema_with_defs = OpenAPISchema { - defs: Some(defs), - ..Default::default() - }; + let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; - let mut schema = OpenAPISchema { - all_of: Some(vec![Box::new(schema_with_defs)]), - ..Default::default() - }; + let mut schema = + OpenAPISchema { all_of: Some(vec![Box::new(schema_with_defs)]), ..Default::default() }; schema.flatten_all_of(); @@ -1343,9 +1391,15 @@ mod tests { schema.sanitize_for_google(); - assert!(schema.schema_url.is_none(), "Root $schema should be removed"); + assert!( + schema.schema_url.is_none(), + "Root $schema should be removed" + ); let field = schema.properties.as_ref().unwrap().get("field").unwrap(); - assert!(field.schema_url.is_none(), "Nested $schema should be removed"); + assert!( + field.schema_url.is_none(), + "Nested $schema should be removed" + ); } #[test] @@ -1365,7 +1419,10 @@ mod tests { schema.sanitize_for_google(); let items = schema.items.as_ref().unwrap(); - assert!(items.schema_url.is_none(), "Array items $schema should be removed"); + assert!( + items.schema_url.is_none(), + "Array items $schema should be removed" + ); } #[test] @@ -1376,15 +1433,16 @@ mod tests { ..Default::default() }; - let mut schema = OpenAPISchema { - one_of: Some(vec![Box::new(variant)]), - ..Default::default() - }; + let mut schema = + OpenAPISchema { one_of: Some(vec![Box::new(variant)]), ..Default::default() }; schema.sanitize_for_google(); let variant = &schema.one_of.as_ref().unwrap()[0]; - assert!(variant.schema_url.is_none(), "oneOf variant $schema should be removed"); + assert!( + variant.schema_url.is_none(), + "oneOf variant $schema should be removed" + ); } #[test] @@ -1405,9 +1463,15 @@ mod tests { schema.sanitize_for_google(); - assert!(schema.schema_url.is_none(), "Root $schema should be removed"); + assert!( + schema.schema_url.is_none(), + "Root $schema should be removed" + ); let my_type = schema.defs.as_ref().unwrap().get("MyType").unwrap(); - assert!(my_type.schema_url.is_none(), "$defs schema $schema should be removed"); + assert!( + my_type.schema_url.is_none(), + "$defs schema $schema should be removed" + ); } #[test] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index c576ad9854..0b820e3816 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -21,7 +21,7 @@ use windmill_mcp::McpClient; #[cfg(not(feature = "mcp"))] use crate::ai::tools::McpClientStub as McpClient; use windmill_common::{ - ai_providers::{AIProvider}, + ai_providers::AIProvider, cache, client::AuthedClient, db::DB, @@ -417,7 +417,7 @@ pub async fn run_agent( args.provider.get_base_url(db).await? }; let api_key = args.provider.get_api_key().unwrap_or(""); - + // Create the query builder for the provider let query_builder = create_query_builder(&args.provider); @@ -666,7 +666,10 @@ pub async fn run_agent( let parsed = if args.provider.kind == AIProvider::AWSBedrock { #[cfg(feature = "bedrock")] { - let region = args.provider.get_region().unwrap_or(windmill_common::ai_providers::USE_ENV_REGION); + let region = args + .provider + .get_region() + .unwrap_or(windmill_common::ai_providers::USE_ENV_REGION); // Use Bedrock SDK via dedicated query builder crate::ai::providers::bedrock::BedrockQueryBuilder::default() .execute_request( @@ -770,10 +773,8 @@ pub async fn run_agent( .build_request_without_usage(&build_args, client, &job.workspace_id) .await?; - let retry_resp = build_http_request(retry_body) - .send() - .await - .map_err(|e| { + let retry_resp = + build_http_request(retry_body).send().await.map_err(|e| { Error::internal_err(format!("Failed to call API on retry: {}", e)) })?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index ebd2805b17..5efd29ff3c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -621,9 +621,21 @@ impl OccupancyMetrics { // long enough to have meaningful data for that window. Otherwise, // short-lived workers would report misleadingly high occupancy rates. ( - if elapsed >= 15.0 { Some(total_occupation_15s) } else { None }, - if elapsed >= 300.0 { Some(total_occupation_5m) } else { None }, - if elapsed >= 1800.0 { Some(total_occupation_30m) } else { None }, + if elapsed >= 15.0 { + Some(total_occupation_15s) + } else { + None + }, + if elapsed >= 300.0 { + Some(total_occupation_5m) + } else { + None + }, + if elapsed >= 1800.0 { + Some(total_occupation_30m) + } else { + None + }, ) } else { (None, None, None) diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 314e1760d3..35f7d10938 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -30,7 +30,8 @@ use crate::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, DEV_CONF_NSJAIL, }, - handle_child::handle_child, get_proxy_envs_for_lang, + get_proxy_envs_for_lang, + handle_child::handle_child, CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 23977e47a2..606ea4ec11 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,11 +7,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, - start_child_process, OccupancyMetrics, StreamNotifier, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, + StreamNotifier, }, + get_proxy_envs_for_lang, handle_child::handle_child, - get_proxy_envs_for_lang, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, + DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -94,7 +96,10 @@ async fn get_common_deno_proc_envs( } // Add proxy envs (including OTEL tracing proxy if enabled for deno) - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno).await.unwrap_or_default() { + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno) + .await + .unwrap_or_default() + { deno_envs.insert(k.to_string(), v); } diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b0475ee14a..d1b2bdff35 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,6 +1,6 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; -use windmill_common::scripts::ScriptLang; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; +use windmill_common::scripts::ScriptLang; use itertools::Itertools; use serde_json::value::RawValue; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 6232f99be2..b3c8bfec28 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -26,8 +26,8 @@ use crate::{ }, handle_child, universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency}, - COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, - JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, + MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 51b1007964..0df576d9b8 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -329,7 +329,11 @@ async fn handle_full_regex( // Use .ok() to match deno_core op_get_id behavior: return null for non-existent steps // instead of throwing an error let res = authed_client - .get_result_by_id::>>(&by_id.flow_job.to_string(), obj_key, query) + .get_result_by_id::>>( + &by_id.flow_job.to_string(), + obj_key, + query, + ) .await .ok() .flatten(); @@ -1393,17 +1397,18 @@ async fn eval_fetch( // Uses job_id as trace_id so all spans are linked to the job. // span_id is a placeholder - it gets overwritten by the OTLP handler with the real parent span_id. #[cfg(all(feature = "private", feature = "enterprise"))] - let otel_context_inject = if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - let trace_id = job_id.as_simple().to_string(); - format!( -r#"globalThis.__enterSpan?.({{ + let otel_context_inject = + if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { + let trace_id = job_id.as_simple().to_string(); + format!( + r#"globalThis.__enterSpan?.({{ isRecording: () => true, spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) }});"# - ) - } else { - String::new() - }; + ) + } else { + String::new() + }; #[cfg(not(all(feature = "private", feature = "enterprise")))] let otel_context_inject = ""; diff --git a/backend/windmill-worker/src/js_eval_parity_tests.rs b/backend/windmill-worker/src/js_eval_parity_tests.rs index 7b403e8ac9..0863598e76 100644 --- a/backend/windmill-worker/src/js_eval_parity_tests.rs +++ b/backend/windmill-worker/src/js_eval_parity_tests.rs @@ -67,9 +67,12 @@ mod parity_tests { let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; assert_eq!( - deno_value, quickjs_value, + deno_value, + quickjs_value, "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, deno_result.get(), quickjs_result.get() + expr, + deno_result.get(), + quickjs_result.get() ); Ok(()) @@ -615,10 +618,7 @@ mod parity_tests { env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([1, 2, 3])))); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1}))), - ); + env.insert("obj".to_string(), Arc::new(to_raw_value(&json!({"a": 1})))); env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); // typeof @@ -746,7 +746,10 @@ mod parity_tests { // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout let mut env = HashMap::new(); - env.insert("status".to_string(), Arc::new(to_raw_value(&json!("pending")))); + env.insert( + "status".to_string(), + Arc::new(to_raw_value(&json!("pending"))), + ); env.insert("retries".to_string(), Arc::new(to_raw_value(&json!(3)))); env.insert("maxRetries".to_string(), Arc::new(to_raw_value(&json!(5)))); @@ -805,20 +808,10 @@ mod parity_tests { test_parity("numbers.map(n => n * 2)", env.clone(), None).await?; // Block body (explicit return) - test_parity( - "numbers.map(n => { return n * 2; })", - env.clone(), - None, - ) - .await?; + test_parity("numbers.map(n => { return n * 2; })", env.clone(), None).await?; // Multiple parameters - test_parity( - "numbers.reduce((acc, n) => acc + n, 0)", - env.clone(), - None, - ) - .await?; + test_parity("numbers.reduce((acc, n) => acc + n, 0)", env.clone(), None).await?; // Destructuring in parameters test_parity( @@ -957,12 +950,7 @@ mod parity_tests { .await?; // Deep clone pattern - test_parity( - "JSON.parse(JSON.stringify(config))", - env.clone(), - None, - ) - .await?; + test_parity("JSON.parse(JSON.stringify(config))", env.clone(), None).await?; // Computed property names test_parity( @@ -1048,12 +1036,7 @@ mod parity_tests { ); // find and findIndex - test_parity( - "items.find(i => i.name === 'Banana')", - env.clone(), - None, - ) - .await?; + test_parity("items.find(i => i.name === 'Banana')", env.clone(), None).await?; test_parity( "items.findIndex(i => i.name === 'Banana')", @@ -1082,28 +1065,13 @@ mod parity_tests { test_parity("Array(3).fill(0)", env.clone(), None).await?; // Reverse (on copy to avoid mutation) - test_parity( - "[...items].reverse().map(i => i.name)", - env.clone(), - None, - ) - .await?; + test_parity("[...items].reverse().map(i => i.name)", env.clone(), None).await?; // concat - test_parity( - "[1, 2].concat([3, 4], [5, 6])", - env.clone(), - None, - ) - .await?; + test_parity("[1, 2].concat([3, 4], [5, 6])", env.clone(), None).await?; // join variations - test_parity( - "items.map(i => i.name).join(' | ')", - env.clone(), - None, - ) - .await?; + test_parity("items.map(i => i.name).join(' | ')", env.clone(), None).await?; Ok(()) } @@ -1289,8 +1257,14 @@ mod parity_tests { async fn parity_edge_cases_empty_values() -> anyhow::Result<()> { let mut env = HashMap::new(); env.insert("emptyArray".to_string(), Arc::new(to_raw_value(&json!([])))); - env.insert("emptyObject".to_string(), Arc::new(to_raw_value(&json!({})))); - env.insert("emptyString".to_string(), Arc::new(to_raw_value(&json!("")))); + env.insert( + "emptyObject".to_string(), + Arc::new(to_raw_value(&json!({}))), + ); + env.insert( + "emptyString".to_string(), + Arc::new(to_raw_value(&json!(""))), + ); env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); // Operations on empty values @@ -1409,19 +1383,9 @@ mod parity_tests { let env = HashMap::new(); // Basic Promise.resolve - test_parity( - "Promise.resolve(42)", - env.clone(), - None, - ) - .await?; + test_parity("Promise.resolve(42)", env.clone(), None).await?; - test_parity( - "Promise.resolve({ key: 'value' })", - env.clone(), - None, - ) - .await?; + test_parity("Promise.resolve({ key: 'value' })", env.clone(), None).await?; // Promise.all with resolved values test_parity( @@ -1447,35 +1411,15 @@ mod parity_tests { ); // Deduplicate using Set - test_parity( - "[...new Set(arr)]", - env.clone(), - None, - ) - .await?; + test_parity("[...new Set(arr)]", env.clone(), None).await?; // Set size - test_parity( - "new Set(arr).size", - env.clone(), - None, - ) - .await?; + test_parity("new Set(arr).size", env.clone(), None).await?; // Set.has - test_parity( - "new Set(arr).has(3)", - env.clone(), - None, - ) - .await?; + test_parity("new Set(arr).has(3)", env.clone(), None).await?; - test_parity( - "new Set(arr).has(99)", - env.clone(), - None, - ) - .await?; + test_parity("new Set(arr).has(99)", env.clone(), None).await?; Ok(()) } @@ -1798,32 +1742,17 @@ mod benchmark_tests { // QuickJS let start = Instant::now(); for _ in 0..iterations { - let _ = eval_timeout_quickjs( - expr.to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; + let _ = + eval_timeout_quickjs(expr.to_string(), env.clone(), None, None, None, None, None) + .await?; } let quickjs_duration = start.elapsed(); // deno_core let start = Instant::now(); for _ in 0..iterations { - let _ = eval_timeout( - expr.to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; + let _ = + eval_timeout(expr.to_string(), env.clone(), None, None, None, None, None).await?; } let deno_duration = start.elapsed(); @@ -1886,9 +1815,12 @@ mod flow_simulation_parity_tests { let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; assert_eq!( - deno_value, quickjs_value, + deno_value, + quickjs_value, "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, deno_result.get(), quickjs_result.get() + expr, + deno_result.get(), + quickjs_result.get() ); Ok(()) @@ -1906,10 +1838,7 @@ mod flow_simulation_parity_tests { let mut transform_context = HashMap::new(); // Step 'a' result: simple number - transform_context.insert( - "a".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); + transform_context.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); // Step 'b' result: object with nested data transform_context.insert( @@ -1938,10 +1867,7 @@ mod flow_simulation_parity_tests { ); // Step 'd' result: null (simulating a step that returned null) - transform_context.insert( - "d".to_string(), - Arc::new(to_raw_value(&json!(null))), - ); + transform_context.insert("d".to_string(), Arc::new(to_raw_value(&json!(null)))); // Step 'e' result: error object (simulating a failed step with continue_on_error) transform_context.insert( @@ -2057,10 +1983,22 @@ mod flow_simulation_parity_tests { test_parity("b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("b.data.users[0].name", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("b.data.users[1].roles", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.metadata.hasMore", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "b.data.metadata.hasMore", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Deeply nested - test_parity("f.level1.level2.level3.level4.value", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "f.level1.level2.level3.level4.value", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2076,13 +2014,25 @@ mod flow_simulation_parity_tests { // Array methods test_parity("c.map(x => x * 2)", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("c.filter(x => x > 25)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.reduce((acc, x) => acc + x, 0)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "c.reduce((acc, x) => acc + x, 0)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; test_parity("c.find(x => x === 30)", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("c.some(x => x > 40)", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("c.every(x => x > 0)", ctx.clone(), fi.clone(), fe.clone()).await?; // Chained operations - test_parity("c.filter(x => x > 20).map(x => x / 10)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "c.filter(x => x > 20).map(x => x / 10)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2094,18 +2044,27 @@ mod flow_simulation_parity_tests { // Complex data extraction from step 'b' test_parity( "b.data.users.filter(u => u.active).map(u => u.name)", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; test_parity( "b.data.users.filter(u => u.roles.includes('admin'))[0]?.name", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; test_parity( "b.data.users.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Combining multiple step results test_parity("a + c[0]", ctx.clone(), fi.clone(), fe.clone()).await?; @@ -2133,10 +2092,34 @@ mod flow_simulation_parity_tests { async fn parity_flow_input_nested() -> anyhow::Result<()> { let (ctx, fi, fe) = create_multi_step_flow_context(); - test_parity("flow_input.config.timeout", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.retries", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.options", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.options[0]", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "flow_input.config.timeout", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.config.retries", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.config.options", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.config.options[0]", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2145,10 +2128,34 @@ mod flow_simulation_parity_tests { async fn parity_flow_input_array_operations() -> anyhow::Result<()> { let (ctx, fi, fe) = create_multi_step_flow_context(); - test_parity("flow_input.items.length", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items[0].id", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items.map(i => i.value)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items.find(i => i.id === 2)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "flow_input.items.length", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.items[0].id", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.items.map(i => i.value)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input.items.find(i => i.id === 2)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2159,13 +2166,22 @@ mod flow_simulation_parity_tests { // Combining flow_input with step results test_parity("flow_input.count + a", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.timeout * b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "flow_input.config.timeout * b.data.total", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Conditional based on flow_input test_parity( "flow_input.enabled ? b.data.users : []", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2189,18 +2205,30 @@ mod flow_simulation_parity_tests { async fn parity_flow_env_conditionals() -> anyhow::Result<()> { // Test flow_env conditionals with explicit flow_env reference in context let mut ctx = HashMap::new(); - ctx.insert("env_val".to_string(), Arc::new(to_raw_value(&json!("production")))); - ctx.insert("debug_val".to_string(), Arc::new(to_raw_value(&json!(false)))); + ctx.insert( + "env_val".to_string(), + Arc::new(to_raw_value(&json!("production"))), + ); + ctx.insert( + "debug_val".to_string(), + Arc::new(to_raw_value(&json!(false))), + ); test_parity( "env_val === 'production' ? 'prod' : 'dev'", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; test_parity( "debug_val ? 'debug mode' : 'normal'", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2214,16 +2242,34 @@ mod flow_simulation_parity_tests { let (ctx, fi, fe) = create_multi_step_flow_context(); // Typical iterator expressions - test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; // Direct array - test_parity("b.data.users", ctx.clone(), fi.clone(), fe.clone()).await?; // Nested array + test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; // Direct array + test_parity("b.data.users", ctx.clone(), fi.clone(), fe.clone()).await?; // Nested array test_parity("flow_input.items", ctx.clone(), fi.clone(), fe.clone()).await?; // Transformed iterators - test_parity("c.map(x => ({value: x, doubled: x * 2}))", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users.filter(u => u.active)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "c.map(x => ({value: x, doubled: x * 2}))", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "b.data.users.filter(u => u.active)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Range-like iteration - test_parity("Array.from({length: 5}, (_, i) => i)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "Array.from({length: 5}, (_, i) => i)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2232,13 +2278,19 @@ mod flow_simulation_parity_tests { async fn parity_forloop_inner_expressions() -> anyhow::Result<()> { // Simulate expressions inside a for-loop where flow_input.iter exists let mut ctx = HashMap::new(); - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!({"value": 42, "index": 2})))); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({"value": 42, "index": 2}))), + ); let mut flow_input = HashMap::new(); - flow_input.insert("iter".to_string(), to_raw_value(&json!({ - "index": 2, - "value": {"id": 3, "name": "test_item"} - }))); + flow_input.insert( + "iter".to_string(), + to_raw_value(&json!({ + "index": 2, + "value": {"id": 3, "name": "test_item"} + })), + ); flow_input.insert("name".to_string(), to_raw_value(&json!("parent_flow"))); let fi = Some(mappable_rc::Marc::new(flow_input)); @@ -2251,8 +2303,11 @@ mod flow_simulation_parity_tests { // Combining iter with other flow_input test_parity( "`Item ${flow_input.iter.index} of ${flow_input.name}`", - ctx.clone(), fi.clone(), None - ).await?; + ctx.clone(), + fi.clone(), + None, + ) + .await?; Ok(()) } @@ -2271,12 +2326,24 @@ mod flow_simulation_parity_tests { test_parity("flow_input.enabled", ctx.clone(), fi.clone(), None).await?; // Complex boolean conditions - test_parity("a > 40 && b.status === 'success'", ctx.clone(), fi.clone(), None).await?; + test_parity( + "a > 40 && b.status === 'success'", + ctx.clone(), + fi.clone(), + None, + ) + .await?; test_parity("a < 50 || b.data.total > 5", ctx.clone(), fi.clone(), None).await?; // Conditions with array checks test_parity("b.data.users.length > 0", ctx.clone(), fi.clone(), None).await?; - test_parity("b.data.users.some(u => u.active)", ctx.clone(), fi.clone(), None).await?; + test_parity( + "b.data.users.some(u => u.active)", + ctx.clone(), + fi.clone(), + None, + ) + .await?; test_parity("c.includes(30)", ctx.clone(), fi.clone(), None).await?; Ok(()) @@ -2291,16 +2358,40 @@ mod flow_simulation_parity_tests { let (ctx, fi, fe) = create_multi_step_flow_context(); // Skip based on previous result - test_parity("previous_result === null", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result.length === 0", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "previous_result === null", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "previous_result.length === 0", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Skip based on flow_input test_parity("!flow_input.enabled", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.count === 0", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "flow_input.count === 0", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Skip based on step result test_parity("d === null", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e?.error !== undefined", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "e?.error !== undefined", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2311,7 +2402,13 @@ mod flow_simulation_parity_tests { // Stop conditions (avoid previous_result?.error pattern which has issues with error extraction) test_parity("a >= 42", ctx.clone(), fi.clone(), None).await?; - test_parity("b.data.metadata.hasMore === false", ctx.clone(), fi.clone(), None).await?; + test_parity( + "b.data.metadata.hasMore === false", + ctx.clone(), + fi.clone(), + None, + ) + .await?; test_parity("b.status !== 'success'", ctx.clone(), fi.clone(), None).await?; Ok(()) @@ -2327,7 +2424,7 @@ mod flow_simulation_parity_tests { let mut ctx = HashMap::new(); ctx.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); // 'b' was never executed (branch not taken) - ctx.insert("c".to_string(), Arc::new(to_raw_value(&json!(null)))); // Step returned null + ctx.insert("c".to_string(), Arc::new(to_raw_value(&json!(null)))); // Step returned null // Safe access to potentially missing step test_parity("a", ctx.clone(), None, None).await?; @@ -2346,10 +2443,22 @@ mod flow_simulation_parity_tests { // Nullish coalescing test_parity("d ?? 'default'", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("d?.value ?? 'not found'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "d?.value ?? 'not found'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // With nested access - test_parity("b.data.missing?.value ?? 'fallback'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "b.data.missing?.value ?? 'fallback'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2370,8 +2479,11 @@ mod flow_simulation_parity_tests { // Conditional based on error test_parity( "e.error ? `Error: ${e.error.message}` : 'OK'", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2431,19 +2543,28 @@ mod flow_simulation_parity_tests { // Building objects from step results test_parity( "({ count: a, users: b.data.users })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; test_parity( "({ ...flow_input.config, extra: 'value' })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Computed properties test_parity( "({ [`step_${a}`]: b.status })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2459,8 +2580,11 @@ mod flow_simulation_parity_tests { // Array from step results test_parity( "[b.data.users[0], b.data.users[2]]", - ctx.clone(), fi.clone(), fe.clone() - ).await?; + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2473,12 +2597,18 @@ mod flow_simulation_parity_tests { async fn parity_multiline_data_processing() -> anyhow::Result<()> { let (ctx, fi, fe) = create_multi_step_flow_context(); - test_parity(r#" + test_parity( + r#" let users = b.data.users; let activeUsers = users.filter(u => u.active); let adminUsers = activeUsers.filter(u => u.roles.includes('admin')); return adminUsers.map(u => u.name); - "#, ctx.clone(), fi.clone(), fe.clone()).await?; + "#, + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2487,13 +2617,19 @@ mod flow_simulation_parity_tests { async fn parity_multiline_conditional_logic() -> anyhow::Result<()> { let (ctx, fi, _fe) = create_multi_step_flow_context(); - test_parity(r#" + test_parity( + r#" if (flow_input.enabled) { return { mode: 'enabled', data: b.data.users.filter(u => u.active) }; } else { return { mode: 'disabled', data: b.data.users }; } - "#, ctx.clone(), fi.clone(), None).await?; + "#, + ctx.clone(), + fi.clone(), + None, + ) + .await?; Ok(()) } @@ -2502,7 +2638,8 @@ mod flow_simulation_parity_tests { async fn parity_multiline_aggregation() -> anyhow::Result<()> { let (ctx, fi, _fe) = create_multi_step_flow_context(); - test_parity(r#" + test_parity( + r#" const summary = { stepA: a, stepB_status: b.status, @@ -2513,7 +2650,12 @@ mod flow_simulation_parity_tests { enabled: flow_input.enabled }; return summary; - "#, ctx.clone(), fi.clone(), None).await?; + "#, + ctx.clone(), + fi.clone(), + None, + ) + .await?; Ok(()) } @@ -2540,8 +2682,14 @@ mod flow_simulation_parity_tests { #[tokio::test] async fn parity_large_numbers() -> anyhow::Result<()> { let mut ctx = HashMap::new(); - ctx.insert("bigInt".to_string(), Arc::new(to_raw_value(&json!(9007199254740991_i64)))); // MAX_SAFE_INTEGER - ctx.insert("timestamp".to_string(), Arc::new(to_raw_value(&json!(1703980800000_i64)))); // Typical timestamp + ctx.insert( + "bigInt".to_string(), + Arc::new(to_raw_value(&json!(9007199254740991_i64))), + ); // MAX_SAFE_INTEGER + ctx.insert( + "timestamp".to_string(), + Arc::new(to_raw_value(&json!(1703980800000_i64))), + ); // Typical timestamp test_parity("bigInt", ctx.clone(), None, None).await?; test_parity("timestamp", ctx.clone(), None, None).await?; @@ -2577,9 +2725,15 @@ mod flow_simulation_parity_tests { async fn parity_boolean_coercion_edge_cases() -> anyhow::Result<()> { let mut ctx = HashMap::new(); ctx.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - ctx.insert("emptyString".to_string(), Arc::new(to_raw_value(&json!("")))); + ctx.insert( + "emptyString".to_string(), + Arc::new(to_raw_value(&json!(""))), + ); ctx.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); - ctx.insert("falseVal".to_string(), Arc::new(to_raw_value(&json!(false)))); + ctx.insert( + "falseVal".to_string(), + Arc::new(to_raw_value(&json!(false))), + ); ctx.insert("emptyArr".to_string(), Arc::new(to_raw_value(&json!([])))); ctx.insert("emptyObj".to_string(), Arc::new(to_raw_value(&json!({})))); @@ -2588,12 +2742,12 @@ mod flow_simulation_parity_tests { test_parity("!!emptyString", ctx.clone(), None, None).await?; test_parity("!!nullVal", ctx.clone(), None, None).await?; test_parity("!!falseVal", ctx.clone(), None, None).await?; - test_parity("!!emptyArr", ctx.clone(), None, None).await?; // [] is truthy! - test_parity("!!emptyObj", ctx.clone(), None, None).await?; // {} is truthy! + test_parity("!!emptyArr", ctx.clone(), None, None).await?; // [] is truthy! + test_parity("!!emptyObj", ctx.clone(), None, None).await?; // {} is truthy! // Logical operators with falsy values test_parity("zero || 'default'", ctx.clone(), None, None).await?; - test_parity("zero ?? 'default'", ctx.clone(), None, None).await?; // 0 is not nullish + test_parity("zero ?? 'default'", ctx.clone(), None, None).await?; // 0 is not nullish test_parity("nullVal ?? 'default'", ctx.clone(), None, None).await?; Ok(()) @@ -2609,8 +2763,20 @@ mod flow_simulation_parity_tests { test_parity("previous_result", ctx.clone(), fi.clone(), fe.clone()).await?; test_parity("previous_result[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result.length", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result[4].key", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "previous_result.length", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "previous_result[4].key", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -2631,23 +2797,26 @@ mod flow_simulation_parity_tests { "hasMore": true }))), ); - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!({ - "items": [{"id": 1}, {"id": 2}, {"id": 3}], - "nextCursor": "abc123", - "hasMore": true - })))); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({ + "items": [{"id": 1}, {"id": 2}, {"id": 3}], + "nextCursor": "abc123", + "hasMore": true + }))), + ); // Iterator for next page test_parity( "previous_result.hasMore ? [previous_result.nextCursor] : []", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; // Accumulating results - test_parity( - "fetch_result.items", - ctx.clone(), None, None - ).await?; + test_parity("fetch_result.items", ctx.clone(), None, None).await?; Ok(()) } @@ -2678,31 +2847,46 @@ mod flow_simulation_parity_tests { ]))), ); - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!([ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"} - ])))); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!([ + {"date": "2024-01-15", "amount": 100, "type": "credit"}, + {"date": "2024-01-17", "amount": 200, "type": "credit"} + ]))), + ); // Filter expression test_parity( "raw_data.records.filter(r => r.type === 'credit')", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; // Sum expression test_parity( "credits.reduce((sum, r) => sum + r.amount, 0)", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; // Summary - test_parity(r#" + test_parity( + r#" ({ totalCredits: credits.reduce((sum, r) => sum + r.amount, 0), count: credits.length, average: credits.reduce((sum, r) => sum + r.amount, 0) / credits.length }) - "#, ctx.clone(), None, None).await?; + "#, + ctx.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2729,20 +2913,29 @@ mod flow_simulation_parity_tests { // Branch condition test_parity( "check_result.passed && check_result.score > 90", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; // Skip condition test_parity( "!check_result.passed || check_result.score < 50", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; // Decision logic test_parity( "check_result.passed && user_data.level === 'admin' ? 'approved' : 'pending'", - ctx.clone(), None, None - ).await?; + ctx.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2764,11 +2957,20 @@ mod flow_simulation_parity_tests { }))), ); env.insert("nullObj".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("undefinedField".to_string(), Arc::new(to_raw_value(&serde_json::Value::Null))); + env.insert( + "undefinedField".to_string(), + Arc::new(to_raw_value(&serde_json::Value::Null)), + ); // Optional chaining on method calls test_parity("obj?.data?.items?.map(x => x * 2)", env.clone(), None, None).await?; - test_parity("obj?.data?.items?.filter(x => x > 1)", env.clone(), None, None).await?; + test_parity( + "obj?.data?.items?.filter(x => x > 1)", + env.clone(), + None, + None, + ) + .await?; test_parity("obj?.data?.items?.join(',')", env.clone(), None, None).await?; test_parity("obj?.data?.name?.toUpperCase()", env.clone(), None, None).await?; test_parity("obj?.data?.name?.split('')", env.clone(), None, None).await?; @@ -2807,10 +3009,22 @@ mod flow_simulation_parity_tests { // Computed access with null/undefined test_parity("nullData?.users?.[key]", env.clone(), None, None).await?; test_parity("data?.missing?.[key]", env.clone(), None, None).await?; - test_parity("data?.users?.['nonexistent']?.name", env.clone(), None, None).await?; + test_parity( + "data?.users?.['nonexistent']?.name", + env.clone(), + None, + None, + ) + .await?; // Dynamic key access - test_parity("data?.users?.[`user${index + 1}`]?.name", env.clone(), None, None).await?; + test_parity( + "data?.users?.[`user${index + 1}`]?.name", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2826,7 +3040,10 @@ mod flow_simulation_parity_tests { "value": 42 }))), ); - env.insert("nullConfig".to_string(), Arc::new(to_raw_value(&json!(null)))); + env.insert( + "nullConfig".to_string(), + Arc::new(to_raw_value(&json!(null))), + ); // Optional function call syntax test_parity("config?.callback?.()", env.clone(), None, None).await?; @@ -2857,16 +3074,49 @@ mod flow_simulation_parity_tests { } }))), ); - env.insert("emptyResponse".to_string(), Arc::new(to_raw_value(&json!({})))); + env.insert( + "emptyResponse".to_string(), + Arc::new(to_raw_value(&json!({}))), + ); // Deep optional chaining - test_parity("response?.data?.result?.items?.[0]?.details?.metadata?.tags", env.clone(), None, None).await?; - test_parity("response?.data?.result?.items?.[0]?.details?.metadata?.tags?.[0]", env.clone(), None, None).await?; - test_parity("response?.data?.result?.items?.[1]?.details?.metadata?.tags", env.clone(), None, None).await?; + test_parity( + "response?.data?.result?.items?.[0]?.details?.metadata?.tags", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "response?.data?.result?.items?.[0]?.details?.metadata?.tags?.[0]", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "response?.data?.result?.items?.[1]?.details?.metadata?.tags", + env.clone(), + None, + None, + ) + .await?; // Deep chaining with missing intermediate - test_parity("emptyResponse?.data?.result?.items?.[0]", env.clone(), None, None).await?; - test_parity("response?.data?.missing?.items?.[0]?.details", env.clone(), None, None).await?; + test_parity( + "emptyResponse?.data?.result?.items?.[0]", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "response?.data?.missing?.items?.[0]?.details", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2889,19 +3139,55 @@ mod flow_simulation_parity_tests { env.insert("nullUser".to_string(), Arc::new(to_raw_value(&json!(null)))); // Optional chaining with nullish coalescing - test_parity("user?.profile?.settings?.theme ?? 'light'", env.clone(), None, None).await?; - test_parity("user?.profile?.settings?.language ?? 'en'", env.clone(), None, None).await?; - test_parity("nullUser?.profile?.theme ?? 'default'", env.clone(), None, None).await?; + test_parity( + "user?.profile?.settings?.theme ?? 'light'", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "user?.profile?.settings?.language ?? 'en'", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "nullUser?.profile?.theme ?? 'default'", + env.clone(), + None, + None, + ) + .await?; // Optional chaining with logical OR - test_parity("user?.profile?.settings?.disabled || false", env.clone(), None, None).await?; + test_parity( + "user?.profile?.settings?.disabled || false", + env.clone(), + None, + None, + ) + .await?; test_parity("user?.name || 'Anonymous'", env.clone(), None, None).await?; // Optional chaining with logical AND - test_parity("user?.profile?.settings?.notifications && 'enabled'", env.clone(), None, None).await?; + test_parity( + "user?.profile?.settings?.notifications && 'enabled'", + env.clone(), + None, + None, + ) + .await?; // Optional chaining in ternary - test_parity("user?.profile?.settings?.theme === 'dark' ? 'Dark Mode' : 'Light Mode'", env.clone(), None, None).await?; + test_parity( + "user?.profile?.settings?.theme === 'dark' ? 'Dark Mode' : 'Light Mode'", + env.clone(), + None, + None, + ) + .await?; test_parity("nullUser?.active ? 'yes' : 'no'", env.clone(), None, None).await?; // Optional chaining with arithmetic @@ -2931,21 +3217,63 @@ mod flow_simulation_parity_tests { env.insert("emptyData".to_string(), Arc::new(to_raw_value(&json!({})))); // Optional chaining before array methods - test_parity("data?.users?.filter(u => u.active)", env.clone(), None, None).await?; + test_parity( + "data?.users?.filter(u => u.active)", + env.clone(), + None, + None, + ) + .await?; test_parity("data?.users?.map(u => u.name)", env.clone(), None, None).await?; - test_parity("data?.users?.find(u => u.id === 2)?.name", env.clone(), None, None).await?; - test_parity("data?.users?.findIndex(u => u.id === 2)", env.clone(), None, None).await?; + test_parity( + "data?.users?.find(u => u.id === 2)?.name", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "data?.users?.findIndex(u => u.id === 2)", + env.clone(), + None, + None, + ) + .await?; test_parity("data?.users?.some(u => u.active)", env.clone(), None, None).await?; test_parity("data?.users?.every(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.users?.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", env.clone(), None, None).await?; + test_parity( + "data?.users?.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", + env.clone(), + None, + None, + ) + .await?; // Optional chaining on missing arrays - test_parity("emptyData?.users?.filter(u => u.active)", env.clone(), None, None).await?; + test_parity( + "emptyData?.users?.filter(u => u.active)", + env.clone(), + None, + None, + ) + .await?; test_parity("data?.items?.map(i => i.value)", env.clone(), None, None).await?; // Chained optional access on array results - test_parity("data?.users?.filter(u => u.active)?.[0]?.name", env.clone(), None, None).await?; - test_parity("data?.users?.filter(u => u.id > 10)?.[0]?.name ?? 'Not found'", env.clone(), None, None).await?; + test_parity( + "data?.users?.filter(u => u.active)?.[0]?.name", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "data?.users?.filter(u => u.id > 10)?.[0]?.name ?? 'Not found'", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2964,13 +3292,34 @@ mod flow_simulation_parity_tests { } }))), ); - env.insert("nullPerson".to_string(), Arc::new(to_raw_value(&json!(null)))); + env.insert( + "nullPerson".to_string(), + Arc::new(to_raw_value(&json!(null))), + ); // Template literals with optional chaining - test_parity("`Hello, ${person?.firstName ?? 'Guest'}!`", env.clone(), None, None).await?; - test_parity("`${person?.firstName} ${person?.lastName}`", env.clone(), None, None).await?; + test_parity( + "`Hello, ${person?.firstName ?? 'Guest'}!`", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "`${person?.firstName} ${person?.lastName}`", + env.clone(), + None, + None, + ) + .await?; test_parity("`Location: ${person?.address?.city ?? 'Unknown'}, ${person?.address?.country ?? 'Unknown'}`", env.clone(), None, None).await?; - test_parity("`User: ${nullPerson?.name ?? 'Anonymous'}`", env.clone(), None, None).await?; + test_parity( + "`User: ${nullPerson?.name ?? 'Anonymous'}`", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -2981,20 +3330,68 @@ mod flow_simulation_parity_tests { // Optional chaining on step results test_parity("a?.toString()", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.[0]?.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.find(u => u.id === 999)?.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.find(u => u.id === 999)?.name ?? 'Not found'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "b?.data?.users?.[0]?.name", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "b?.data?.users?.find(u => u.id === 999)?.name", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "b?.data?.users?.find(u => u.id === 999)?.name ?? 'Not found'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Optional chaining on missing nested properties (step 'b' exists but nested path may not) - test_parity("b?.missing?.nested?.value ?? 'default'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "b?.missing?.nested?.value ?? 'default'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Optional chaining on flow_input - test_parity("flow_input?.limit ?? 100", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input?.missing?.nested?.value ?? 'fallback'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "flow_input?.limit ?? 100", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "flow_input?.missing?.nested?.value ?? 'fallback'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Optional chaining on previous_result - test_parity("previous_result?.items?.[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result?.missing ?? []", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "previous_result?.items?.[0]", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "previous_result?.missing ?? []", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -3004,7 +3401,10 @@ mod flow_simulation_parity_tests { let mut env = HashMap::new(); env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); env.insert("emptyStr".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert("falseVal".to_string(), Arc::new(to_raw_value(&json!(false)))); + env.insert( + "falseVal".to_string(), + Arc::new(to_raw_value(&json!(false))), + ); env.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); env.insert( "nested".to_string(), @@ -3032,7 +3432,13 @@ mod flow_simulation_parity_tests { // Empty object access test_parity("nested?.obj?.missing", env.clone(), None, None).await?; - test_parity("nested?.obj?.missing ?? 'not there'", env.clone(), None, None).await?; + test_parity( + "nested?.obj?.missing ?? 'not there'", + env.clone(), + None, + None, + ) + .await?; // Chaining after primitives (should return undefined) test_parity("nested?.zero?.value", env.clone(), None, None).await?; @@ -3146,15 +3552,39 @@ mod flow_simulation_parity_tests { test_parity("[1, 2, 3].length", env.clone(), None, None).await?; test_parity("[1, 2, 3].map(x => x * 2)", env.clone(), None, None).await?; test_parity("[1, 2, 3].filter(x => x > 1)", env.clone(), None, None).await?; - test_parity("[1, 2, 3].reduce((a, b) => a + b, 0)", env.clone(), None, None).await?; + test_parity( + "[1, 2, 3].reduce((a, b) => a + b, 0)", + env.clone(), + None, + None, + ) + .await?; // Array with undefined values (different from holes) - test_parity("[1, undefined, 3].map(x => x ?? 'missing')", env.clone(), None, None).await?; - test_parity("[1, null, 3].map(x => x ?? 'missing')", env.clone(), None, None).await?; + test_parity( + "[1, undefined, 3].map(x => x ?? 'missing')", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "[1, null, 3].map(x => x ?? 'missing')", + env.clone(), + None, + None, + ) + .await?; // Array.from behavior test_parity("Array.from([1, 2, 3])", env.clone(), None, None).await?; - test_parity("Array.from({length: 3}, (_, i) => i)", env.clone(), None, None).await?; + test_parity( + "Array.from({length: 3}, (_, i) => i)", + env.clone(), + None, + None, + ) + .await?; // Spread operator test_parity("[...arr]", env.clone(), None, None).await?; @@ -3167,18 +3597,12 @@ mod flow_simulation_parity_tests { #[tokio::test] async fn parity_unicode_and_emoji() -> anyhow::Result<()> { let mut env = HashMap::new(); - env.insert( - "emoji".to_string(), - Arc::new(to_raw_value(&json!("🎉"))), - ); + env.insert("emoji".to_string(), Arc::new(to_raw_value(&json!("🎉")))); env.insert( "text_with_emoji".to_string(), Arc::new(to_raw_value(&json!("Hello 🌍 World!"))), ); - env.insert( - "cafe".to_string(), - Arc::new(to_raw_value(&json!("café"))), - ); + env.insert("cafe".to_string(), Arc::new(to_raw_value(&json!("café")))); env.insert( "chinese".to_string(), Arc::new(to_raw_value(&json!("你好世界"))), @@ -3222,10 +3646,7 @@ mod flow_simulation_parity_tests { "negative_zero_str".to_string(), Arc::new(to_raw_value(&json!("-0"))), ); - env.insert( - "num".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); + env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); // Basic numeric operations test_parity("0 === 0", env.clone(), None, None).await?; @@ -3244,8 +3665,20 @@ mod flow_simulation_parity_tests { // Safe integer checks test_parity("Number.isSafeInteger(42)", env.clone(), None, None).await?; - test_parity("Number.isSafeInteger(9007199254740991)", env.clone(), None, None).await?; - test_parity("Number.isSafeInteger(9007199254740992)", env.clone(), None, None).await?; + test_parity( + "Number.isSafeInteger(9007199254740991)", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "Number.isSafeInteger(9007199254740992)", + env.clone(), + None, + None, + ) + .await?; // Number parsing test_parity("parseInt('42')", env.clone(), None, None).await?; @@ -3281,8 +3714,20 @@ mod flow_simulation_parity_tests { // Object.keys, Object.values, Object.entries // Note: Order might differ but we compare as sets test_parity("Object.keys(obj).sort()", env.clone(), None, None).await?; - test_parity("Object.values(obj).sort((a, b) => a - b)", env.clone(), None, None).await?; - test_parity("Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))", env.clone(), None, None).await?; + test_parity( + "Object.values(obj).sort((a, b) => a - b)", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))", + env.clone(), + None, + None, + ) + .await?; // Object spread (order might differ) test_parity("{...obj, extra: 5}", env.clone(), None, None).await?; @@ -3373,7 +3818,13 @@ mod flow_simulation_parity_tests { test_parity("text.match(/hello/i)", env.clone(), None, None).await?; // Email validation (basic pattern) - test_parity("/^[^@]+@[^@]+\\.[^@]+$/.test(email)", env.clone(), None, None).await?; + test_parity( + "/^[^@]+@[^@]+\\.[^@]+$/.test(email)", + env.clone(), + None, + None, + ) + .await?; // Capturing groups (basic) test_parity("text.match(/(\\d+)/)", env.clone(), None, None).await?; @@ -3399,19 +3850,49 @@ mod flow_simulation_parity_tests { test_parity("new Date(timestamp).toISOString()", env.clone(), None, None).await?; // UTC methods (timezone-independent) - test_parity("new Date(iso_date).getUTCFullYear()", env.clone(), None, None).await?; + test_parity( + "new Date(iso_date).getUTCFullYear()", + env.clone(), + None, + None, + ) + .await?; test_parity("new Date(iso_date).getUTCMonth()", env.clone(), None, None).await?; test_parity("new Date(iso_date).getUTCDate()", env.clone(), None, None).await?; test_parity("new Date(iso_date).getUTCHours()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCMinutes()", env.clone(), None, None).await?; + test_parity( + "new Date(iso_date).getUTCMinutes()", + env.clone(), + None, + None, + ) + .await?; // Date arithmetic - test_parity("new Date(timestamp + 86400000).toISOString()", env.clone(), None, None).await?; - test_parity("new Date(timestamp - 3600000).toISOString()", env.clone(), None, None).await?; + test_parity( + "new Date(timestamp + 86400000).toISOString()", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "new Date(timestamp - 3600000).toISOString()", + env.clone(), + None, + None, + ) + .await?; // Date comparison test_parity("new Date(iso_date).getTime() > 0", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getTime() === Date.parse(iso_date)", env.clone(), None, None).await?; + test_parity( + "new Date(iso_date).getTime() === Date.parse(iso_date)", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3439,8 +3920,20 @@ mod flow_simulation_parity_tests { .await?; // Typeof for error prevention - test_parity("typeof b.missing === 'undefined'", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("typeof b.data.total === 'number'", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "typeof b.missing === 'undefined'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "typeof b.data.total === 'number'", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Ternary with type checks test_parity( @@ -3490,14 +3983,8 @@ mod flow_simulation_parity_tests { "key".to_string(), Arc::new(to_raw_value(&json!("dynamicKey"))), ); - env.insert( - "prefix".to_string(), - Arc::new(to_raw_value(&json!("item"))), - ); - env.insert( - "index".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); + env.insert("prefix".to_string(), Arc::new(to_raw_value(&json!("item")))); + env.insert("index".to_string(), Arc::new(to_raw_value(&json!(42)))); // Computed property access test_parity("({a: 1, b: 2})[key] ?? 'missing'", env.clone(), None, None).await?; @@ -3506,7 +3993,13 @@ mod flow_simulation_parity_tests { // Computed property creation test_parity("({[key]: 'value'})", env.clone(), None, None).await?; test_parity("({[prefix + '_' + index]: true})", env.clone(), None, None).await?; - test_parity("({[`${prefix}_${index}`]: 'computed'})", env.clone(), None, None).await?; + test_parity( + "({[`${prefix}_${index}`]: 'computed'})", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3525,15 +4018,39 @@ mod flow_simulation_parity_tests { // Nested destructuring test_parity("(({user: {name}}) => name)(data)", env.clone(), None, None).await?; - test_parity("(({items: [first, second, ...rest]}) => ({first, second, rest}))(data)", env.clone(), None, None).await?; + test_parity( + "(({items: [first, second, ...rest]}) => ({first, second, rest}))(data)", + env.clone(), + None, + None, + ) + .await?; // Default values in destructuring - test_parity("(({missing = 'default'}) => missing)(data)", env.clone(), None, None).await?; - test_parity("(({user: {nickname = 'unknown'}}) => nickname)(data)", env.clone(), None, None).await?; + test_parity( + "(({missing = 'default'}) => missing)(data)", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "(({user: {nickname = 'unknown'}}) => nickname)(data)", + env.clone(), + None, + None, + ) + .await?; // Renaming in destructuring test_parity("(({user: u}) => u.name)(data)", env.clone(), None, None).await?; - test_parity("(({meta: {count: total}}) => total)(data)", env.clone(), None, None).await?; + test_parity( + "(({meta: {count: total}}) => total)(data)", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3557,7 +4074,13 @@ mod flow_simulation_parity_tests { // Simple arrow functions test_parity("numbers.map(x => x * 2)", env.clone(), None, None).await?; test_parity("numbers.filter(x => x > 2)", env.clone(), None, None).await?; - test_parity("numbers.reduce((a, b) => a + b, 0)", env.clone(), None, None).await?; + test_parity( + "numbers.reduce((a, b) => a + b, 0)", + env.clone(), + None, + None, + ) + .await?; // Arrow functions with objects test_parity("users.map(u => u.name)", env.clone(), None, None).await?; @@ -3565,15 +4088,45 @@ mod flow_simulation_parity_tests { test_parity("users.find(u => u.name === 'Bob')", env.clone(), None, None).await?; // Arrow functions returning objects (note the parentheses) - test_parity("numbers.map(x => ({value: x, doubled: x * 2}))", env.clone(), None, None).await?; + test_parity( + "numbers.map(x => ({value: x, doubled: x * 2}))", + env.clone(), + None, + None, + ) + .await?; // Chained arrow function calls - test_parity("numbers.filter(x => x > 1).map(x => x * 10)", env.clone(), None, None).await?; - test_parity("users.filter(u => u.score > 80).map(u => u.name)", env.clone(), None, None).await?; + test_parity( + "numbers.filter(x => x > 1).map(x => x * 10)", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "users.filter(u => u.score > 80).map(u => u.name)", + env.clone(), + None, + None, + ) + .await?; // Arrow function with multiple params - test_parity("numbers.reduce((sum, val) => sum + val, 0)", env.clone(), None, None).await?; - test_parity("numbers.map((val, idx) => ({index: idx, value: val}))", env.clone(), None, None).await?; + test_parity( + "numbers.reduce((sum, val) => sum + val, 0)", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "numbers.map((val, idx) => ({index: idx, value: val}))", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3583,8 +4136,14 @@ mod flow_simulation_parity_tests { let mut env = HashMap::new(); env.insert("str_num".to_string(), Arc::new(to_raw_value(&json!("42")))); env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("bool_true".to_string(), Arc::new(to_raw_value(&json!(true)))); - env.insert("bool_false".to_string(), Arc::new(to_raw_value(&json!(false)))); + env.insert( + "bool_true".to_string(), + Arc::new(to_raw_value(&json!(true))), + ); + env.insert( + "bool_false".to_string(), + Arc::new(to_raw_value(&json!(false))), + ); env.insert("null_val".to_string(), Arc::new(to_raw_value(&json!(null)))); env.insert("empty_str".to_string(), Arc::new(to_raw_value(&json!("")))); env.insert("empty_arr".to_string(), Arc::new(to_raw_value(&json!([])))); @@ -3650,8 +4209,20 @@ mod flow_simulation_parity_tests { test_parity("JSON.parse(json_str).count", env.clone(), None, None).await?; // Round-trip - test_parity("JSON.parse(JSON.stringify(obj)).name", env.clone(), None, None).await?; - test_parity("JSON.parse(JSON.stringify(obj)).values", env.clone(), None, None).await?; + test_parity( + "JSON.parse(JSON.stringify(obj)).name", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "JSON.parse(JSON.stringify(obj)).values", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3661,7 +4232,10 @@ mod flow_simulation_parity_tests { let mut env = HashMap::new(); env.insert("x".to_string(), Arc::new(to_raw_value(&json!(16)))); env.insert("y".to_string(), Arc::new(to_raw_value(&json!(-5.7)))); - env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9])))); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9]))), + ); // Basic Math functions test_parity("Math.abs(y)", env.clone(), None, None).await?; @@ -3679,8 +4253,20 @@ mod flow_simulation_parity_tests { test_parity("Math.max(...arr)", env.clone(), None, None).await?; // Trigonometric (with rounding to avoid precision issues) - test_parity("Math.round(Math.sin(0) * 1000) / 1000", env.clone(), None, None).await?; - test_parity("Math.round(Math.cos(0) * 1000) / 1000", env.clone(), None, None).await?; + test_parity( + "Math.round(Math.sin(0) * 1000) / 1000", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "Math.round(Math.cos(0) * 1000) / 1000", + env.clone(), + None, + None, + ) + .await?; // Logarithmic test_parity("Math.log(1)", env.clone(), None, None).await?; @@ -3752,7 +4338,13 @@ mod flow_simulation_parity_tests { // Copy and splice (to avoid mutating original) test_parity("[...arr].splice(1, 2)", env.clone(), None, None).await?; - test_parity("(() => { const a = [...arr]; a.splice(1, 2, 'x'); return a; })()", env.clone(), None, None).await?; + test_parity( + "(() => { const a = [...arr]; a.splice(1, 2, 'x'); return a; })()", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3774,13 +4366,37 @@ mod flow_simulation_parity_tests { ); // Find error in step h's results - test_parity("h.find(r => r.error)?.error", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("h.filter(r => r.error).length", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "h.find(r => r.error)?.error", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; + test_parity( + "h.filter(r => r.error).length", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; test_parity("h.some(r => r.error)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("h.every(r => !r.error)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "h.every(r => !r.error)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; // Extract all successful results - test_parity("h.filter(r => r.success).map(r => r.data)", ctx.clone(), fi.clone(), fe.clone()).await?; + test_parity( + "h.filter(r => r.success).map(r => r.data)", + ctx.clone(), + fi.clone(), + fe.clone(), + ) + .await?; Ok(()) } @@ -3802,17 +4418,41 @@ mod flow_simulation_parity_tests { ); // Nested expressions in templates - test_parity("`User: ${user.name} (${user.email})`", env.clone(), None, None).await?; + test_parity( + "`User: ${user.name} (${user.email})`", + env.clone(), + None, + None, + ) + .await?; test_parity("`Score: ${user.score.toFixed(1)}`", env.clone(), None, None).await?; test_parity("`Items: ${items.join(', ')}`", env.clone(), None, None).await?; test_parity("`Count: ${items.length}`", env.clone(), None, None).await?; // Conditional in template - test_parity("`Status: ${user.score >= 90 ? 'A' : 'B'}`", env.clone(), None, None).await?; + test_parity( + "`Status: ${user.score >= 90 ? 'A' : 'B'}`", + env.clone(), + None, + None, + ) + .await?; // Method calls in template - test_parity("`Upper: ${user.name.toUpperCase()}`", env.clone(), None, None).await?; - test_parity("`First item: ${items[0].charAt(0).toUpperCase() + items[0].slice(1)}`", env.clone(), None, None).await?; + test_parity( + "`Upper: ${user.name.toUpperCase()}`", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "`First item: ${items[0].charAt(0).toUpperCase() + items[0].slice(1)}`", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -3842,10 +4482,7 @@ mod flow_simulation_parity_tests { #[tokio::test] async fn parity_es2022_string_at() -> anyhow::Result<()> { let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello"))), - ); + env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); // String.prototype.at() - ES2022 test_parity("str.at(0)", env.clone(), None, None).await?; @@ -3989,7 +4626,13 @@ mod flow_simulation_parity_tests { ); // Object.groupBy() - ES2024 - test_parity("Object.groupBy(items, item => item.type)", env.clone(), None, None).await?; + test_parity( + "Object.groupBy(items, item => item.type)", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -4037,7 +4680,13 @@ mod flow_simulation_parity_tests { ); // Named capture groups - may not work in QuickJS - test_parity("/(?\\d{4})-(?\\d{2})-(?\\d{2})/.exec(date)?.groups?.year", env.clone(), None, None).await?; + test_parity( + "/(?\\d{4})-(?\\d{2})-(?\\d{2})/.exec(date)?.groups?.year", + env.clone(), + None, + None, + ) + .await?; Ok(()) } @@ -4068,8 +4717,20 @@ mod flow_simulation_parity_tests { test_parity("typeof atob", env.clone(), None, None).await?; test_parity("typeof btoa", env.clone(), None, None).await?; // If available, test actual usage - test_parity("typeof btoa === 'function' ? btoa('hello') : 'not_available'", env.clone(), None, None).await?; - test_parity("typeof atob === 'function' ? atob('aGVsbG8=') : 'not_available'", env.clone(), None, None).await?; + test_parity( + "typeof btoa === 'function' ? btoa('hello') : 'not_available'", + env.clone(), + None, + None, + ) + .await?; + test_parity( + "typeof atob === 'function' ? atob('aGVsbG8=') : 'not_available'", + env.clone(), + None, + None, + ) + .await?; Ok(()) } diff --git a/backend/windmill-worker/src/js_eval_quickjs.rs b/backend/windmill-worker/src/js_eval_quickjs.rs index 2dc48c753a..7c59e36ced 100644 --- a/backend/windmill-worker/src/js_eval_quickjs.rs +++ b/backend/windmill-worker/src/js_eval_quickjs.rs @@ -139,9 +139,7 @@ pub async fn eval_timeout_quickjs( ) .await .map_err(|_| { - anyhow::anyhow!( - "The expression evaluation `{expr}` took too long to execute (>10000ms)" - ) + anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)") })?? } @@ -328,7 +326,9 @@ fn setup_async_ops<'js>( .get_resource_value_interpolated::(&path, None) .await { - Ok(value) => serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()), + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } Err(e) => format!("{}{}", ERR_PREFIX, e), } } @@ -419,27 +419,40 @@ fn setup_results_proxy<'js>( Some(jr) => { // Found in local cache, fetch result by job ID match jr { - JobResult::SingleJob(job_id) => { - client - .get_completed_job_result::(&job_id.to_string(), None) - .await - .map_err(|e| format!("Failed to fetch result for step '{}': {}", step_id_clone, e)) - } + JobResult::SingleJob(job_id) => client + .get_completed_job_result::( + &job_id.to_string(), + None, + ) + .await + .map_err(|e| { + format!( + "Failed to fetch result for step '{}': {}", + step_id_clone, e + ) + }), JobResult::ListJob(job_ids) => { let futs = job_ids.iter().map(|job_id| { let client = client.clone(); let job_id_str = job_id.to_string(); async move { client - .get_completed_job_result::(&job_id_str, None) + .get_completed_job_result::( + &job_id_str, + None, + ) .await } }); let results: Vec<_> = futures::future::join_all(futs).await; - let collected: Result, _> = results.into_iter().collect(); - collected - .map(serde_json::Value::Array) - .map_err(|e| format!("Failed to fetch results for step '{}': {}", step_id_clone, e)) + let collected: Result, _> = + results.into_iter().collect(); + collected.map(serde_json::Value::Array).map_err(|e| { + format!( + "Failed to fetch results for step '{}': {}", + step_id_clone, e + ) + }) } } } @@ -449,15 +462,21 @@ fn setup_results_proxy<'js>( // Use .ok() to match deno_core behavior: return null for non-existent steps // instead of throwing an error Ok(client - .get_result_by_id::(&flow_job_id, &step_id_clone, None) + .get_result_by_id::( + &flow_job_id, + &step_id_clone, + None, + ) .await - .ok() // Swallow errors, convert to Option - .unwrap_or(serde_json::Value::Null)) // None -> null + .ok() // Swallow errors, convert to Option + .unwrap_or(serde_json::Value::Null)) // None -> null } }; match result { - Ok(value) => serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()), + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } Err(e) => format!("{}{}", ERR_PREFIX, e), } } @@ -577,8 +596,24 @@ fn should_add_return_quickjs(expr: &str) -> bool { } let statement_prefixes = [ - "const ", "let ", "var ", "if ", "if(", "for ", "for(", "while ", "while(", "switch ", - "switch(", "try ", "try{", "throw ", "function ", "class ", "async ", "await ", + "const ", + "let ", + "var ", + "if ", + "if(", + "for ", + "for(", + "while ", + "while(", + "switch ", + "switch(", + "try ", + "try{", + "throw ", + "function ", + "class ", + "async ", + "await ", ]; for prefix in &statement_prefixes { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index fed4e8d31b..caf2b84ec3 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -34,18 +34,15 @@ mod global_cache; mod go_executor; mod graphql_executor; mod handle_child; -#[cfg(all(feature = "private", feature = "enterprise"))] -mod otel_tracing_proxy_ee; -mod otel_tracing_proxy_oss; pub mod job_logger; #[cfg(feature = "private")] pub mod job_logger_ee; mod job_logger_oss; mod js_eval; -#[cfg(feature = "quickjs")] -pub mod js_eval_quickjs; #[cfg(test)] mod js_eval_parity_tests; +#[cfg(feature = "quickjs")] +pub mod js_eval_quickjs; pub mod memory_common; #[cfg(feature = "private")] pub mod memory_ee; @@ -59,9 +56,13 @@ mod oracledb_executor; #[cfg(feature = "private")] pub mod otel_ee; mod otel_oss; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod otel_tracing_proxy_ee; +mod otel_tracing_proxy_oss; mod pg_executor; #[cfg(feature = "php")] mod php_executor; +mod prepare_deps; #[cfg(feature = "python")] mod python_executor; #[cfg(feature = "python")] @@ -71,33 +72,23 @@ pub mod result_processor; mod rust_executor; mod sanitized_sql_params; mod schema; -pub mod scoped_dependency_map; pub mod sql_utils; mod universal_pkg_installer; -mod prepare_deps; mod worker; mod worker_flow; mod worker_lockfiles; mod worker_utils; -pub mod workspace_dependencies; -pub use worker::*; -pub use worker_lockfiles::{ - process_relative_imports, trigger_dependents_to_recompute_dependencies, -}; #[cfg(all(feature = "private", feature = "enterprise"))] -pub use otel_tracing_proxy_ee::{ - set_current_job_context, start_jobs_otel_tracing, TRACING_PROXY_PORT, -}; +pub use otel_tracing_proxy_ee::start_jobs_otel_tracing; #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] -pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED, OTLP_COLLECTOR_PORT}; - -pub use result_processor::handle_job_error; +pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED}; +pub use worker::*; pub use bun_executor::{ build_loader, compute_bundle_local_and_remote_path, generate_dedicated_worker_wrapper, get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, - BUN_DEDICATED_WORKER_ARGS, LoaderMode, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, + LoaderMode, BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, }; pub use deno_executor::generate_deno_lock; pub use prepare_deps::run_prepare_deps_cli; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 954c844209..99f67ed456 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -396,7 +396,10 @@ fn string_date_to_mysql_date(s: &str) -> mysql_async::Value { get_capture_by_index(&caps, 1), get_capture_by_index(&caps, 2), get_capture_by_index(&caps, 3), - 0, 0, 0, 0, + 0, + 0, + 0, + 0, ); } diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 7e17678426..9fc947fc70 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,11 +16,11 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - handle_child, get_proxy_envs_for_lang, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + get_proxy_envs_for_lang, handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; -use windmill_common::scripts::ScriptLang; use windmill_common::client::AuthedClient; +use windmill_common::scripts::ScriptLang; const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto"); lazy_static::lazy_static! { diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index f02f114280..69eaed0425 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -22,7 +22,8 @@ use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, python_executor::{INDEX_CERT, NATIVE_CERT, PYTHON_PATH, UV_PATH}, - HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, WIN_ENVS, + HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, + WIN_ENVS, }; impl From for PyVAlias { @@ -527,9 +528,18 @@ impl PyV { .env("HOME", HOME_ENV.to_string()) .env("PATH", PATH_ENV.to_string()) .envs(PROXY_ENVS.clone()) - .args(["python", "install", &v, "--python-preference=only-managed", "--no-bin"]) + .args([ + "python", + "install", + &v, + "--python-preference=only-managed", + "--no-bin", + ]) // TODO: Do we need these? - .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), ("UV_CACHE_DIR", UV_CACHE_DIR)]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), + ("UV_CACHE_DIR", UV_CACHE_DIR), + ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index f0762a6e0d..9cd656f9e3 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -26,7 +26,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - handle_child::{self}, get_proxy_envs_for_lang, + get_proxy_envs_for_lang, + handle_child::{self}, universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, TRACING_PROXY_CA_CERT_PATH, @@ -51,7 +52,6 @@ const NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT: &str = include_str!("../nsjail/download.ruby.config.proto"); const NSJAIL_CONFIG_LOCK_RUBY_CONTENT: &str = include_str!("../nsjail/lock.ruby.config.proto"); - #[allow(dead_code)] pub(crate) struct JobHandlerInput<'a> { pub base_internal_url: &'a str, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 90695145f8..4c195ca24e 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,15 +19,17 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, OccupancyMetrics, + DEV_CONF_NSJAIL, }, + get_proxy_envs_for_lang, handle_child::handle_child, - get_proxy_envs_for_lang, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - PROXY_ENVS, RUST_CACHE_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, + TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; -use windmill_common::scripts::ScriptLang; use windmill_common::client::AuthedClient; +use windmill_common::scripts::ScriptLang; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -53,7 +55,6 @@ lazy_static::lazy_static! { static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); } - #[cfg(not(windows))] lazy_static::lazy_static! { static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); diff --git a/backend/windmill-worker/src/schema.rs b/backend/windmill-worker/src/schema.rs index eca9afa72e..f73dcc7a21 100644 --- a/backend/windmill-worker/src/schema.rs +++ b/backend/windmill-worker/src/schema.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use windmill_common::schema::{SchemaValidationRule, SchemaValidator}; use windmill_parser::{MainArgSignature, Typ}; - fn make_rules_for_arg_typ(typ: &Typ) -> Vec { let mut rules = vec![]; @@ -66,7 +65,10 @@ fn make_rules_for_arg_typ(typ: &Typ) -> Vec { for prop in &variant.properties { obj_rules.push((prop.key.to_string(), make_rules_for_arg_typ(&prop.typ))); } - rules_map.insert(variant.label.to_string(), vec![SchemaValidationRule::IsObject(obj_rules)]); + rules_map.insert( + variant.label.to_string(), + vec![SchemaValidationRule::IsObject(obj_rules)], + ); } rules.push(SchemaValidationRule::IsOneOf(rules_map)) @@ -94,4 +96,3 @@ pub fn schema_validator_from_main_arg_sig(sig: &MainArgSignature) -> SchemaValid SchemaValidator { required, rules } } - diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index e1dad165c0..4c68d76135 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -119,11 +119,16 @@ async fn poll_snowflake_async_query( })?; let status = response.status(); - let body = response.text().await.map_err(|e| { - Error::ExecutionErr(format!("error reading poll response body: {}", e)) - })?; + let body = response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("error reading poll response body: {}", e)))?; - tracing::debug!("Snowflake poll response status: {}, body: {}", status, &body[..body.len().min(500)]); + tracing::debug!( + "Snowflake poll response status: {}, body: {}", + status, + &body[..body.len().min(500)] + ); if status == reqwest::StatusCode::ACCEPTED { // Still running, wait and poll again @@ -243,13 +248,14 @@ fn do_snowflake_inner<'a>( let body = raw_response.text().await.map_err(|e| { Error::ExecutionErr(format!("error reading response body: {}", e)) })?; - let async_resp: SnowflakeAsyncResponse = serde_json::from_str(&body).map_err(|e| { - Error::ExecutionErr(format!( - "error decoding async response: {}. Body preview: {}", - e, - &body[..body.len().min(500)] - )) - })?; + let async_resp: SnowflakeAsyncResponse = + serde_json::from_str(&body).map_err(|e| { + Error::ExecutionErr(format!( + "error decoding async response: {}. Body preview: {}", + e, + &body[..body.len().min(500)] + )) + })?; tracing::info!( "Snowflake statement running asynchronously, polling for completion (handle: {})", @@ -273,21 +279,27 @@ fn do_snowflake_inner<'a>( // Handle both sync (200) and async (202) responses let raw_response = handle_snowflake_result(result).await?; let status = raw_response.status(); - let body = raw_response.text().await.map_err(|e| { - Error::ExecutionErr(format!("error reading response body: {}", e)) - })?; + let body = raw_response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("error reading response body: {}", e)))?; - tracing::debug!("Snowflake response status: {}, body: {}", status, &body[..body.len().min(1000)]); + tracing::debug!( + "Snowflake response status: {}, body: {}", + status, + &body[..body.len().min(1000)] + ); let response = if status == reqwest::StatusCode::ACCEPTED { // Async execution - need to poll for results - let async_resp: SnowflakeAsyncResponse = serde_json::from_str(&body).map_err(|e| { - Error::ExecutionErr(format!( - "error decoding async response: {}. Body preview: {}", - e, - &body[..body.len().min(500)] - )) - })?; + let async_resp: SnowflakeAsyncResponse = + serde_json::from_str(&body).map_err(|e| { + Error::ExecutionErr(format!( + "error decoding async response: {}. Body preview: {}", + e, + &body[..body.len().min(500)] + )) + })?; tracing::info!( "Snowflake query running asynchronously, polling for results (handle: {})", diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 982e74c25d..8991df34d7 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -131,12 +131,11 @@ use crate::{ go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, - handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, pwsh_executor::handle_powershell_job, - result_processor::{process_result, start_background_processor}, + result_processor::{handle_job_error, process_result, start_background_processor}, schema::schema_validator_from_main_arg_sig, worker_flow::{handle_flow, SchedulePushZombieError}, worker_lockfiles::{ diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d3f2a39eae..6ecae4f636 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -21,6 +21,7 @@ use crate::{ use anyhow::Context; use async_once_cell::Lazy; +use backon::{BackoffBuilder, ConstantBuilder, Retryable}; use futures::TryFutureExt; use mappable_rc::Marc; use serde::{Deserialize, Serialize}; @@ -30,7 +31,6 @@ use sqlx::types::Json; use sqlx::{FromRow, Postgres, Transaction}; use tracing::instrument; use uuid::Uuid; -use backon::{BackoffBuilder, ConstantBuilder, Retryable}; use windmill_common::auth::get_job_perms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; @@ -68,9 +68,10 @@ use windmill_common::{ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - try_schedule_next_job, insert_concurrency_key, interpolate_args, - report_error_to_workspace_handler_or_critical_side_channel, CanceledBy, FlowRunners, - MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + insert_concurrency_key, interpolate_args, + report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, + FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, + WrappedError, }; use windmill_audit::audit_oss::audit_log; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 0209eec361..5da446c825 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1,20 +1,16 @@ use std::borrow::Cow; use std::collections::HashMap; use std::fs::{create_dir_all, remove_dir_all}; -use std::path::{Component, Path, PathBuf}; #[cfg(feature = "python")] use crate::ansible_executor::{get_git_repos_lock, AnsibleDependencyLocks}; -use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; use async_recursion::async_recursion; -use chrono::{Duration, Utc}; use itertools::Itertools; use serde::Serialize; use serde_json::value::RawValue; use serde_json::{from_value, json, Value}; use sha2::Digest; use sqlx::types::Json; -use tokio::time::timeout; use uuid::Uuid; use windmill_common::assets::{ clear_static_asset_usage, insert_static_asset_usage, AssetUsageKind, @@ -22,17 +18,15 @@ use windmill_common::assets::{ use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; -use windmill_common::jobs::JobPayload; -use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; use windmill_common::scripts::ScriptHash; -use windmill_common::utils::WarnAfterExt; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; -use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ - RawWorkspaceDependencies, WorkspaceDependencies, WorkspaceDependenciesPrefetched, + RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -44,13 +38,12 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +pub use windmill_dep_map::{ + extract_referenced_paths, extract_relative_imports, process_relative_imports, +}; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; -#[cfg(feature = "python")] -use windmill_parser_py_imports::parse_relative_imports; -use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{ - append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel, - WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, + append_logs, CanceledBy, MiniPulledJob, WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, }; // TODO: To be removed in future versions @@ -59,9 +52,6 @@ lazy_static::lazy_static! { static ref WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_NEW_APP_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_APP_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_COMPONENTS_TO_RELOCK: bool = std::env::var("WMDEBUG_NO_COMPONENTS_TO_RELOCK").is_ok(); - static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( - if cfg!(test) { /* if test we want increased debouncing delay */ 15 } else { 5 } - ); } use crate::common::{MaybeLock, OccupancyMetrics}; @@ -84,103 +74,6 @@ use crate::{ go_executor::install_go_dependencies, }; -fn try_normalize(path: &Path) -> Option { - let mut ret = PathBuf::new(); - - for component in path.components() { - match component { - Component::Prefix(..) | Component::RootDir => return None, - Component::CurDir => {} - Component::ParentDir => { - if !ret.pop() { - return None; - } - } - Component::Normal(c) => { - ret.push(c); - } - } - } - - Some(ret) -} - -fn parse_ts_relative_imports(raw_code: &str, script_path: &str) -> error::Result> { - let mut relative_imports = vec![]; - let r = parse_expr_for_imports(raw_code, true)?; - for import in r { - let import = import.trim_end_matches(".ts"); - if import.starts_with("/") { - relative_imports.push(import.trim_start_matches("/").to_string()); - } else if import.starts_with(".") { - let normalized = try_normalize(std::path::Path::new(&format!( - "{}/../{}", - script_path, import - ))); - if let Some(normalized) = normalized { - let normalized = normalized.to_str().unwrap().to_string(); - relative_imports.push(normalized); - } else { - tracing::error!("error canonicalizing path: {script_path} with import {import}"); - } - } - } - - Ok(relative_imports) -} - -pub fn extract_relative_imports( - raw_code: &str, - script_path: &str, - language: &Option, -) -> Option> { - match language { - #[cfg(feature = "python")] - Some(ScriptLang::Python3) => parse_relative_imports(&raw_code, script_path).ok(), - Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) | Some(ScriptLang::Deno) => { - parse_ts_relative_imports(&raw_code, script_path).ok() - } - _ => None, - } -} - -pub fn extract_referenced_paths( - raw_code: &str, - script_path: &str, - language: Option, -) -> Option> { - let mut referenced_paths = vec![]; - if let Some(wk_deps_refs) = language - .and_then(|l| l.extract_workspace_dependencies_annotated_refs(raw_code, script_path)) - .map(|r| r.external) - { - let l = language.expect("should be some"); - for wk_deps_ref in wk_deps_refs { - if let Some(path) = WorkspaceDependencies::to_path(&Some(wk_deps_ref), l).ok() { - referenced_paths.push(path); - }; - } - } else if let (Some(l), true /* Only if it is not blacklisted */) = ( - language, - WorkspaceDependenciesPrefetched::is_external_references_permitted(script_path), - ) { - // we assume all runnables without annotated dependencies reference default dependencies file. - WorkspaceDependencies::to_path(&None, l) - .ok() - .inspect(|p| referenced_paths.push(p.to_owned())); - } - - if let Some(relative_imports) = extract_relative_imports(raw_code, script_path, &language) { - referenced_paths.extend(relative_imports); - } - - if referenced_paths.is_empty() { - None - } else { - Some(referenced_paths) - } -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_dependency_job( job: &MiniPulledJob, @@ -364,315 +257,6 @@ fn remove_ansi_codes(s: &str) -> String { ANSI_REGEX.replace_all(s, "").to_string() } -pub async fn process_relative_imports( - db: &sqlx::Pool, - _job_id: Option, - args: Option<&Json>>>, - w_id: &str, - script_path: &str, - parent_path: Option, - deployment_message: Option, - code: &str, - script_lang: &Option, - permissioned_as_email: &str, - created_by: &str, - permissioned_as: &str, -) -> error::Result<()> { - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } - - { - let mut already_visited = args - .map(|x| { - x.get("already_visited") - .map(|v| serde_json::from_str::>(v.get()).ok()) - .flatten() - }) - .flatten() - .unwrap_or_default(); - - // TODO: There is a race-condition. - // This can be old version. - - // Check lines of code below, you will find that we get the latest version of the script/app/flow - - // However the latest version does not necessarily mean that it is finalized. - // Instead we assume that this would be the version we would base on. - - // So the script_importers might be behind. Thus some information like nodes_to_relock might be lost. - let importers = crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( - script_path, - w_id, - db, - ) - .await?; - - already_visited.push(script_path.to_string()); - // But currently we will do this extra db call for every script regardless of whether they have relative imports or not - // Script might have no relative imports but still be referenced by someone else. - match timeout( - core::time::Duration::from_secs(60), - Box::pin(trigger_dependents_to_recompute_dependencies( - w_id, - importers, - deployment_message, - parent_path, - permissioned_as_email, - created_by, - permissioned_as, - db, - already_visited, - )), - ) - .warn_after_seconds(10) - .await - { - Ok(Err(e)) => { - tracing::error!(%e, "error triggering dependents to recompute dependencies") - } - Err(e) => { - tracing::error!(%e, "triggering dependents to recompute dependencies has timed out") - } - _ => {} - } - } - - Ok(()) -} - -pub async fn trigger_dependents_to_recompute_dependencies( - w_id: &str, - importers: Vec, - // imported_path: &str, - deployment_message: Option, - parent_path: Option, - email: &str, - created_by: &str, - permissioned_as: &str, - db: &sqlx::Pool, - already_visited: Vec, -) -> error::Result<()> { - tracing::debug!( - "Triggering dependents to recompute dependencies: {}", - importers.iter().map(|dd| &dd.importer_path).join(",") - ); - for DependencyDependent { importer_path, importer_kind, importer_node_ids } in importers.iter() - { - tracing::trace!("Processing dependency: {:?}", importer_path); - if already_visited.contains(importer_path) { - tracing::trace!("Skipping already visited dependency"); - continue; - } - - let mut tx = db.clone().begin().await?; - let mut args: HashMap> = HashMap::new(); - if let Some(ref dm) = deployment_message { - args.insert("deployment_message".to_string(), to_raw_value(&dm)); - } - if let Some(ref p_path) = parent_path { - // NOTE: - // it's not used but maybe one day it will be useful. allows more back-compatibility for the workers when we need it - // also very useful for debugging/observability - // it adds that information to the job args so you can see from the runs page - args.insert("common_dependency_path".to_string(), to_raw_value(&p_path)); - } - - args.insert( - "already_visited".to_string(), - to_raw_value(&already_visited), - ); - - args.insert( - "triggered_by_relative_import".to_string(), - to_raw_value(&true), - ); - - let mut debouncing_settings = DebouncingSettings { - debounce_key: Some(format!("{w_id}:{importer_path}:dependency")), - debounce_delay_s: Some(5), - ..Default::default() - }; - - let job_payload = match importer_kind.as_str() { - // TODO: Make it query only non-archived - // Scripts - "script" => match sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(hash) => { - tracing::debug!("newest hash for {} is: {hash}", importer_path); - - let info = - windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; - - JobPayload::Dependencies { - path: importer_path.clone(), - hash: ScriptHash(hash), - language: info.language, - dedicated_worker: info.dedicated_worker, - debouncing_settings, - } - } - None => { - ScopedDependencyMap::clear_map_for_item( - importer_path, - w_id, - "script", - tx, - &None, - ) - .await - .commit() - .await?; - continue; - } - }, - - // Flows - "flow" => match sqlx::query_scalar!( - "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(version) => { - tracing::debug!("Handling flow dependency update for: {}", importer_path); - - args.insert( - "nodes_to_relock".to_string(), - to_raw_value(&importer_node_ids), - ); - - debouncing_settings.debounce_args_to_accumulate = Some(vec!["nodes_to_relock".into()]); - - JobPayload::FlowDependencies { - path: importer_path.clone(), - version, - dedicated_worker: None, - debouncing_settings, - } - } - None => { - ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "flow", tx, &None) - .await - .commit() - .await?; - continue; - } - }, - - // Apps - "app" => match sqlx::query_scalar!( - "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(version) => { - tracing::debug!("Handling app dependency update for: {}", importer_path); - - args.insert( - "components_to_relock".to_string(), - // TODO: unsafe. Importer Node Ids are not checked. They can simply be array of empty strings! - to_raw_value(importer_node_ids), - ); - - debouncing_settings.debounce_args_to_accumulate = Some(vec!["components_to_relock".into()]); - - JobPayload::AppDependencies { path: importer_path.clone(), version, debouncing_settings } - } - None => { - ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "app", tx, &None) - .await - .commit() - .await?; - continue; - } - }, - - _ => { - tracing::error!( - "unexpected importer kind: {kind:?} for path {path}", - kind = importer_kind, - path = importer_path - ); - continue; - } - }; - - tracing::debug!("Pushing dependency job for: {}", importer_path); - let (job_uuid, new_tx) = windmill_queue::push( - db, - PushIsolationLevel::Transaction(tx), - &w_id, - job_payload, - windmill_queue::PushArgs { args: &args, extra: None }, - &created_by, - email, - permissioned_as.to_string(), - Some("trigger.dependents.to.recompute.dependencies"), - // Schedule for future for debouncing. - Some(Utc::now() + Duration::seconds(*DEPENDENCY_JOB_DEBOUNCE_DELAY as i64)), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - ) - .await?; - - tracing::info!( - "pushed dependency job due to common python path: {job_uuid} for path {path}", - path = importer_path, - ); - new_tx.commit().await?; - } - Ok(()) -} - pub async fn handle_flow_dependency_job( job: MiniPulledJob, preview_data: Option<&RawData>,