From 95f4d15ffe2b85d3712f01946a4b5f2afa92cc5f Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Sat, 14 Dec 2024 13:21:28 +0100 Subject: [PATCH] more feature flags (#4918) * more feature flags * include python in tests * smtp feature flag * del * add back file * fix openapi int --- .github/workflows/backend-test.yml | 2 +- .github/workflows/build-publish-rh-image.yml | 4 +- .github/workflows/build-staging-image.yml | 2 +- .github/workflows/build_windows_worker_.yml | 2 +- .github/workflows/docker-image-rpi4.yml | 2 +- .github/workflows/docker-image.yml | 6 +- .github/workflows/publish_windows_worker.yml | 2 +- backend/Cargo.lock | 2 - backend/Cargo.toml | 11 +- backend/windmill-api/Cargo.toml | 9 +- backend/windmill-api/openapi.yaml | 2 +- backend/windmill-api/src/lib.rs | 46 ++-- backend/windmill-api/src/workers.rs | 2 +- backend/windmill-indexer/src/lib.rs | 2 +- backend/windmill-worker/Cargo.toml | 10 +- .../windmill-worker/src/dedicated_worker.rs | 8 + backend/windmill-worker/src/job_logger_ee.rs | 2 +- backend/windmill-worker/src/lib.rs | 9 +- backend/windmill-worker/src/php_executor.rs | 4 - .../windmill-worker/src/python_executor.rs | 19 +- backend/windmill-worker/src/worker.rs | 62 ++++- .../windmill-worker/src/worker_lockfiles.rs | 240 ++++++++++-------- 22 files changed, 275 insertions(+), 173 deletions(-) diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cbcf40ebe6..77ab92c651 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -49,4 +49,4 @@ jobs: windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features - enterprise,deno_core --all -- --nocapture + enterprise,deno_core,python --all -- --nocapture diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index 67b84f66ac..4cca83cc19 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build-staging-image.yml b/.github/workflows/build-staging-image.yml index cbb6d77572..20d72a51fe 100644 --- a/.github/workflows/build-staging-image.yml +++ b/.github/workflows/build-staging-image.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,csharp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 0fa179eaeb..175e2f6ecf 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -45,7 +45,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp - name: Rename binary with corresponding architecture run: | diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 8e8f43b118..b89c241f20 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,deno_core,php,mysql,csharp + features=embedding,parquet,openidconnect,deno_core,php,mysql,mssql,bigquery,websocket,python,smtp,csharp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a85aaac967..086c22f680 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -76,7 +76,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql,csharp + features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -138,7 +138,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -200,7 +200,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,csharp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql,mssql,bigquery,websocket,python,smtp,csharp PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 67d3d3f0bb..00cdf8267b 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -47,7 +47,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,csharp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql,mssql,bigquery,websocket,python,smtp,csharp - name: Rename binary with corresponding architecture run: | diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c6fa48840c..a0ecb81c33 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10887,7 +10887,6 @@ dependencies = [ "windmill-git-sync", "windmill-indexer", "windmill-parser", - "windmill-parser-py-imports", "windmill-parser-ts", "windmill-queue", ] @@ -11304,7 +11303,6 @@ dependencies = [ "nix", "object_store", "once_cell", - "openidconnect", "opentelemetry", "pem 3.0.4", "postgres-native-tls", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e331343bda..a80dc2bfea 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -47,20 +47,20 @@ lto = "thin" [features] default = [] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml"] stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"] loki = ["windmill-common/loki"] embedding = ["windmill-api/embedding"] -parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"] +parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"] flow_testing = ["windmill-worker/flow_testing"] openidconnect = ["windmill-api/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] -tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] +tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:deno_core"] kafka = ["windmill-api/kafka"] @@ -68,6 +68,11 @@ otel = ["windmill-common/otel", "windmill-worker/otel"] dind = ["windmill-worker/dind"] php = ["windmill-worker/php"] mysql = ["windmill-worker/mysql"] +mssql = ["windmill-worker/mssql"] +bigquery = ["windmill-worker/bigquery"] +websocket = ["windmill-api/websocket"] +python = ["windmill-worker/python"] +smtp = ["windmill-api/smtp"] csharp = ["windmill-worker/csharp"] [dependencies] diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index b9968a75c8..efba9c1e00 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:openidconnect"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"] stripe = ["dep:async-stripe"] enterprise_saml = ["dep:samael"] benchmark = [] @@ -20,13 +20,14 @@ prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:pr openidconnect = ["dep:openidconnect"] tantivy = ["dep:windmill-indexer"] kafka = ["dep:rdkafka"] +websocket = ["dep:tokio-tungstenite"] +smtp = ["dep:mail-parser"] [dependencies] windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true windmill-parser.workspace = true -windmill-parser-py-imports.workspace = true windmill-parser-ts.workspace = true windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } @@ -64,7 +65,7 @@ time.workspace = true native-tls.workspace = true tokio-native-tls.workspace = true openssl.workspace = true -mail-parser = { workspace = true, features = ["serde_support"] } +mail-parser = { workspace = true, features = ["serde_support"], optional = true } magic-crypt.workspace = true tempfile.workspace = true tokio-util.workspace = true @@ -95,7 +96,7 @@ openidconnect = { workspace = true, optional = true} url = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit.workspace = true -tokio-tungstenite.workspace = true +tokio-tungstenite = { workspace = true, optional = true} rdkafka = { workspace = true, optional = true } const_format.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index af8b1c8c74..f5d5d91186 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1126,7 +1126,7 @@ paths: in: query required: false schema: - type: int + type: integer responses: "200": description: new token diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 088707327b..46a5e245ce 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -12,6 +12,7 @@ use crate::ee::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; use crate::oauth2_ee::AllClients; +#[cfg(feature = "smtp")] use crate::smtp_server_ee::SmtpServer; use crate::tracing_init::MyOnFailure; use crate::{ @@ -83,7 +84,8 @@ mod scim_ee; mod scripts; mod service_logs; mod settings; -pub mod smtp_server_ee; +#[cfg(feature = "smtp")] +mod smtp_server_ee; mod static_assets; mod stripe_ee; mod tracing_init; @@ -93,6 +95,7 @@ mod users_ee; mod utils; mod variables; mod webhook_util; +#[cfg(feature = "websocket")] mod websocket_triggers; mod workers; mod workspaces; @@ -214,14 +217,17 @@ pub async fn run_server( #[cfg(feature = "embedding")] load_embeddings_db(&db); - let smtp_server = Arc::new(SmtpServer { - db: db.clone(), - user_db: user_db, - auth_cache: auth_cache.clone(), - base_internal_url: base_internal_url.clone(), - }); - if let Err(err) = smtp_server.start_listener_thread(addr).await { - tracing::error!("Error starting SMTP server: {err:#}"); + #[cfg(feature = "smtp")] + { + let smtp_server = Arc::new(SmtpServer { + db: db.clone(), + user_db: user_db, + auth_cache: auth_cache.clone(), + base_internal_url: base_internal_url.clone(), + }); + if let Err(err) = smtp_server.start_listener_thread(addr).await { + tracing::error!("Error starting SMTP server: {err:#}"); + } } } @@ -253,8 +259,11 @@ pub async fn run_server( }; if !*CLOUD_HOSTED { - let ws_killpill_rx = rx.resubscribe(); - websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await; + #[cfg(feature = "websocket")] + { + let ws_killpill_rx = rx.resubscribe(); + websocket_triggers::start_websockets(db.clone(), ws_killpill_rx).await; + } #[cfg(all(feature = "enterprise", feature = "kafka"))] { @@ -304,10 +313,17 @@ pub async fn run_server( .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_ee::workspaced_service()) .nest("/http_triggers", http_triggers::workspaced_service()) - .nest( - "/websocket_triggers", - websocket_triggers::workspaced_service(), - ) + .nest("/websocket_triggers", { + #[cfg(feature = "websocket")] + { + websocket_triggers::workspaced_service() + } + + #[cfg(not(feature = "websocket"))] + { + Router::new() + } + }) .nest("/kafka_triggers", kafka_triggers_service), ) .nest("/workspaces", workspaces::global_service()) diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 6eeeb17bac..527b66296c 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -185,4 +185,4 @@ async fn get_queue_counts( require_super_admin(&db, &authed.email).await?; let queue_counts = windmill_common::queue::get_queue_counts(&db).await; Ok(Json(queue_counts)) -} \ No newline at end of file +} diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 14c6e7906d..59c6a627f7 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -1,3 +1,3 @@ pub mod completed_runs_ee; -pub mod service_logs_ee; pub mod indexer_ee; +pub mod service_logs_ee; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index dd6099cc9b..86ee7d5e06 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -11,7 +11,9 @@ path = "src/lib.rs" [features] default = [] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"] +mssql = ["dep:tiberius"] +bigquery = ["dep:gcp_auth"] benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = [] parquet = ["windmill-common/parquet", "dep:object_store"] @@ -24,6 +26,7 @@ otel = ["windmill-common/otel", "dep:opentelemetry"] dind = ["dep:bollard"] php = ["dep:windmill-parser-php"] mysql = ["dep:mysql_async"] +python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"] csharp = ["dep:windmill-parser-csharp"] [dependencies] @@ -35,9 +38,9 @@ windmill-parser-ts.workspace = true windmill-parser-go.workspace = true windmill-parser-rust.workspace = true windmill-parser-csharp = { workspace = true, optional = true } -windmill-parser-py.workspace = true +windmill-parser-py = { workspace = true, optional = true } windmill-parser-yaml.workspace = true -windmill-parser-py-imports.workspace = true +windmill-parser-py-imports = { workspace = true, optional = true } windmill-parser-bash.workspace = true windmill-parser-sql.workspace = true windmill-parser-graphql.workspace = true @@ -93,7 +96,6 @@ reqwest.workspace = true hex.workspace = true tiberius = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } -openidconnect = { workspace = true, optional = true} tar.workspace = true object_store = { workspace = true, optional = true} convert_case.workspace = true diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index ad1333c28d..1adac0edf2 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -679,6 +679,14 @@ async fn spawn_dedicated_worker( if let Err(e) = match language { Some(ScriptLang::Python3) => { + #[cfg(not(feature = "python"))] + { + tracing::error!("Python requires the python feature to be enabled"); + killpill_tx.send(()).expect("send"); + return; + } + + #[cfg(feature = "python")] crate::python_executor::start_worker( lock, &db, diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_ee.rs index 15bd4f83b2..e94b163a95 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_ee.rs @@ -37,4 +37,4 @@ pub(crate) fn process_streaming_log_lines( _job_id: &Uuid, ) -> Option> { r.transpose() -} +} \ No newline at end of file diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index c68f77917f..2337fa54d2 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -1,10 +1,11 @@ -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", feature = "bigquery"))] mod bigquery_executor; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", feature = "mssql"))] mod mssql_executor; #[cfg(feature = "enterprise")] mod snowflake_executor; +#[cfg(feature = "python")] mod ansible_executor; mod bash_executor; @@ -19,18 +20,20 @@ mod go_executor; mod graphql_executor; mod handle_child; mod job_logger; +mod job_logger_ee; mod js_eval; #[cfg(feature = "mysql")] mod mysql_executor; mod pg_executor; +#[cfg(feature = "php")] mod php_executor; +#[cfg(feature = "python")] mod python_executor; mod result_processor; mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; -mod job_logger_ee; mod csharp_executor; pub use worker::*; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index cb6b08f4e8..387e2a6566 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -192,13 +192,9 @@ pub async fn handle_php_job( let main_override = get_main_override(job.args.as_ref()); let write_wrapper_f = async { - #[cfg(feature = "php")] let args = windmill_parser_php::parse_php_signature(inner_content, main_override.clone())?.args; - #[cfg(not(feature = "php"))] - let args: Vec = vec![]; - let args_to_include = args .iter() .filter(|x| { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 09caf863e2..233c44fc89 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1518,8 +1518,8 @@ pub async fn handle_python_reqs( if mem < 0 { tracing::warn!( workspace_id = %w_id_2, - "Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}", - pid_o, + "Cannot get memory peak for pid: {:?}, job_id: {:?}, exit code: {mem}", + pid_o, job_id_2 ); } else { @@ -1536,12 +1536,12 @@ pub async fn handle_python_reqs( } else { tracing::debug!( workspace_id = %w_id_2, - "Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}", + "Local mem_peak {:?}mb is smaller then global one {:?}mb, ignoring. job_id: {:?}", local_mem_peak / 1000, *mem_peak_lock / 1000, job_id_2 ); - + } // Get the copy of value and drop lock itself, to release it as fast as possible *mem_peak_lock @@ -1576,28 +1576,27 @@ pub async fn handle_python_reqs( if canceled { tracing::info!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "cancelling installations", ); if let Err(ref e) = kill_tx.send(()){ tracing::error!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "failed to send done: Probably receiving end closed too early or have not opened yet\n{}", // If there is no listener, it will be dropped safely e ); } - } + } } // Once done_tx is dropped, this will be fired _ = done_rx.recv() => break } } }); - } // tl = total_length @@ -1647,7 +1646,9 @@ pub async fn handle_python_reqs( let total_time = std::time::Instant::now(); let has_work = req_with_penv.len() > 0; - for ((i, (req, venv_p)), mut kill_rx) in req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) { + for ((i, (req, venv_p)), mut kill_rx) in + req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) + { let permit = semaphore.clone().acquire_owned().await; // Acquire a permit if let Err(_) = permit { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 396607505b..254bad1c03 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -90,7 +90,6 @@ use tokio::{ use rand::Rng; use crate::{ - ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ @@ -106,8 +105,6 @@ use crate::{ job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, - php_executor::handle_php_job, - python_executor::handle_python_job, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, worker_flow::{handle_flow, update_flow_status_in_progress}, @@ -116,6 +113,15 @@ use crate::{ }, }; +#[cfg(feature = "php")] +use crate::php_executor::handle_php_job; + +#[cfg(feature = "python")] +use crate::python_executor::handle_python_job; + +#[cfg(feature = "python")] +use crate::ansible_executor::handle_ansible_job; + #[cfg(feature = "mysql")] use crate::mysql_executor::do_mysql; @@ -126,9 +132,13 @@ use backon::{BackoffBuilder, Retryable}; use crate::dedicated_worker::create_dedicated_worker_map; #[cfg(feature = "enterprise")] -use crate::{ - bigquery_executor::do_bigquery, mssql_executor::do_mssql, snowflake_executor::do_snowflake, -}; +use crate::snowflake_executor::do_snowflake; + +#[cfg(all(feature = "enterprise", feature = "mssql"))] +use crate::mssql_executor::do_mssql; + +#[cfg(all(feature = "enterprise", feature = "bigquery"))] +use crate::bigquery_executor::do_bigquery; #[cfg(feature = "benchmark")] use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; @@ -2376,7 +2386,9 @@ async fn handle_code_execution_job( .await; } else if language == Some(ScriptLang::Mysql) { #[cfg(not(feature = "mysql"))] - return Err(Error::InternalErr("MySQL requires the mysql feature to be enabled".to_string())); + return Err(Error::InternalErr( + "MySQL requires the mysql feature to be enabled".to_string(), + )); #[cfg(feature = "mysql")] return do_mysql( @@ -2399,7 +2411,14 @@ async fn handle_code_execution_job( )); } - #[cfg(feature = "enterprise")] + #[cfg(not(feature = "bigquery"))] + { + return Err(Error::InternalErr( + "Bigquery requires the bigquery feature to be enabled".to_string(), + )); + } + + #[cfg(all(feature = "enterprise", feature = "bigquery"))] { return do_bigquery( job, @@ -2445,7 +2464,14 @@ async fn handle_code_execution_job( )); } - #[cfg(feature = "enterprise")] + #[cfg(not(feature = "mssql"))] + { + return Err(Error::InternalErr( + "Microsoft SQL server requires the mssql feature to be enabled".to_string(), + )); + } + + #[cfg(all(feature = "enterprise", feature = "mssql"))] { return do_mssql( job, @@ -2549,6 +2575,12 @@ mount {{ ))?; } Some(ScriptLang::Python3) => { + #[cfg(not(feature = "python"))] + return Err(Error::InternalErr( + "Python requires the python feature to be enabled".to_string(), + )); + + #[cfg(feature = "python")] handle_python_job( requirements_o, job_dir, @@ -2660,6 +2692,12 @@ mount {{ .await } Some(ScriptLang::Php) => { + #[cfg(not(feature = "php"))] + return Err(Error::InternalErr( + "PHP requires the php feature to be enabled".to_string(), + )); + + #[cfg(feature = "php")] handle_php_job( requirements_o, mem_peak, @@ -2696,6 +2734,12 @@ mount {{ .await } Some(ScriptLang::Ansible) => { + #[cfg(not(feature = "python"))] + return Err(Error::InternalErr( + "Ansible requires the python feature to be enabled".to_string(), + )); + + #[cfg(feature = "python")] handle_ansible_job( requirements_o, job_dir, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 82baf71030..2e1972065e 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -23,21 +23,24 @@ use windmill_common::{ DB, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +#[cfg(feature = "python")] use windmill_parser_py_imports::parse_relative_imports; use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; use crate::csharp_executor::generate_nuget_lockfile; + +#[cfg(feature = "php")] +use crate::php_executor::{composer_install, parse_php_imports}; +#[cfg(feature = "python")] use crate::python_executor::{ create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL, }; use crate::rust_executor::generate_cargo_lockfile; use crate::{ - bun_executor::gen_bun_lockfile, - deno_executor::generate_deno_lock, + bun_executor::gen_bun_lockfile, deno_executor::generate_deno_lock, go_executor::install_go_dependencies, - php_executor::{composer_install, parse_php_imports}, }; pub async fn update_script_dependency_map( @@ -199,6 +202,7 @@ pub fn extract_relative_imports( language: &Option, ) -> Option> { match language { + #[cfg(feature = "python")] Some(ScriptLang::Python3) => parse_relative_imports(&raw_code, script_path).ok(), Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) => { parse_bun_relative_imports(&raw_code, script_path).ok() @@ -1566,6 +1570,7 @@ pub async fn handle_app_dependency_job( } } +#[cfg(feature = "python")] async fn python_dep( reqs: String, job_id: &Uuid, @@ -1643,95 +1648,111 @@ async fn capture_dependency_job( ) -> error::Result { match job_language { ScriptLang::Python3 => { - let reqs = if raw_deps { - job_raw_code.to_string() - } else { - let mut already_visited = vec![]; + #[cfg(not(feature = "python"))] + return Err(Error::InternalErr( + "Python requires the python feature to be enabled".to_string(), + )); - windmill_parser_py_imports::parse_python_imports( - job_raw_code, - &w_id, - script_path, - &db, - &mut already_visited, - ) - .await? - .join("\n") - }; + #[cfg(feature = "python")] + { + let reqs = if raw_deps { + job_raw_code.to_string() + } else { + let mut already_visited = vec![]; - let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = - PythonAnnotations::parse(job_raw_code); + windmill_parser_py_imports::parse_python_imports( + job_raw_code, + &w_id, + script_path, + &db, + &mut already_visited, + ) + .await? + .join("\n") + }; - if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { - if let Err(e) = sqlx::query!( - r#" - INSERT INTO metrics (id, value) - VALUES ('no_uv_usage_py', $1) - "#, - serde_json::to_value("").map_err(to_anyhow)? - ) - .execute(db) - .await - { - tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e); + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = + PythonAnnotations::parse(job_raw_code); + + if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_py', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e); + } } - } - python_dep( - reqs, - job_id, - mem_peak, - canceled_by, - job_dir, - db, - worker_name, - w_id, - worker_dir, - &mut Some(occupancy_metrics), - no_uv_compile | no_uv, - no_uv_install | no_uv, - ) - .await + python_dep( + reqs, + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + no_uv_compile | no_uv, + no_uv_install | no_uv, + ) + .await + } } ScriptLang::Ansible => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for ansible".to_string(), - )); - } - let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; - let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); + #[cfg(not(feature = "python"))] + return Err(Error::InternalErr( + "Ansible requires the python feature to be enabled".to_string(), + )); - if *USE_PIP_COMPILE || *USE_PIP_INSTALL { - if let Err(e) = sqlx::query!( - r#" - INSERT INTO metrics (id, value) - VALUES ('no_uv_usage_ansible', $1) - "#, - serde_json::to_value("").map_err(to_anyhow)? + #[cfg(feature = "python")] + { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for ansible".to_string(), + )); + } + let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; + let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); + + if *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_ansible', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e); + }; + } + + python_dep( + reqs, + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + false, + false, ) - .execute(db) .await - { - tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e); - }; } - - python_dep( - reqs, - job_id, - mem_peak, - canceled_by, - job_dir, - db, - worker_name, - w_id, - worker_dir, - &mut Some(occupancy_metrics), - false, - false, - ) - .await } ScriptLang::Go => { if raw_deps { @@ -1822,33 +1843,40 @@ async fn capture_dependency_job( Ok(req.unwrap_or_else(String::new)) } ScriptLang::Php => { - let reqs = if raw_deps { - if job_raw_code.is_empty() { - return Ok("".to_string()); - } - job_raw_code.to_string() - } else { - match parse_php_imports(job_raw_code)? { - Some(reqs) => reqs, - None => { + #[cfg(not(feature = "php"))] + return Err(Error::InternalErr( + "PHP requires the php feature to be enabled".to_string(), + )); + + #[cfg(feature = "php")] + { + let reqs = if raw_deps { + if job_raw_code.is_empty() { return Ok("".to_string()); } - } - }; - - composer_install( - mem_peak, - canceled_by, - job_id, - w_id, - db, - job_dir, - worker_name, - reqs, - None, - occupancy_metrics, - ) - .await + job_raw_code.to_string() + } else { + match parse_php_imports(job_raw_code)? { + Some(reqs) => reqs, + None => { + return Ok("".to_string()); + } + } + }; + composer_install( + mem_peak, + canceled_by, + job_id, + w_id, + db, + job_dir, + worker_name, + reqs, + None, + occupancy_metrics, + ) + .await + } } ScriptLang::Rust => { if raw_deps {