diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 84b04f7bb9..5266c78d16 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -49,6 +49,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "atlassian" => "atlassian-python-api", "mysql" => "mysql-connector-python", "tenable" => "pytenable", + "ns1" => "ns1-python", }; fn replace_import(x: String) -> String { diff --git a/backend/src/main.rs b/backend/src/main.rs index 49a564cb66..b3025bef6c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -24,7 +24,7 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, S3_CACHE_CONFIG_SETTING, SAML_METADATA_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, }, stats_ee::schedule_stats, @@ -36,6 +36,9 @@ use windmill_common::{ #[cfg(feature = "enterprise")] use windmill_common::METRICS_ADDR; +#[cfg(feature = "parquet")] +use windmill_common::global_settings::S3_CACHE_CONFIG_SETTING; + use windmill_worker::{ BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, @@ -48,10 +51,13 @@ use crate::monitor::{ monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_s3_cache_setting, reload_scim_token_setting, + reload_retention_period_setting, reload_scim_token_setting, reload_server_config, reload_worker_config, }; +#[cfg(feature = "parquet")] +use crate::monitor::reload_s3_cache_setting; + const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 703f25ec9f..8e5a374ab9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -25,9 +25,9 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, S3_CACHE_CONFIG_SETTING, SAML_METADATA_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, - }, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, s3_helpers::{build_s3_client_from_settings, S3Settings, S3_CACHE_SETTINGS}, server::load_server_config, users::truncate_token, worker::{ + }, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_server_config, users::truncate_token, worker::{ load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG, WORKER_CONFIG, }, BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED @@ -39,6 +39,12 @@ use windmill_worker::{ PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; +#[cfg(feature = "parquet")] +use windmill_common::s3_helpers::{build_s3_client_from_settings, S3_CACHE_SETTINGS, S3Settings}; + +#[cfg(feature = "parquet")] +use windmill_common::global_settings::S3_CACHE_CONFIG_SETTING; + #[cfg(feature = "enterprise")] use crate::ee::verify_license_key; diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index fce1121dc8..4843f63101 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -31,7 +31,9 @@ use windmill_common::{ }; pub fn global_service() -> Router { - Router::new() + + #[warn(unused_mut)] + let mut r = Router::new() .route("/envs", get(get_local_settings)) .route( "/global/:key", @@ -39,8 +41,14 @@ pub fn global_service() -> Router { ) .route("/test_smtp", post(test_email)) .route("/test_license_key", post(test_license_key)) - .route("/test_s3_config", post(test_s3_bucket)) - .route("/send_stats", post(send_stats)) + .route("/send_stats", post(send_stats)); + + #[cfg(feature = "parquet")] + { + r = r.route("/test_s3_config", post(test_s3_bucket)); + } + + return r } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 1b0faa1214..30bf888064 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -1,7 +1,10 @@ #[cfg(feature = "parquet")] use crate::error; +#[cfg(feature = "parquet")] use aws_sdk_sts::config::ProvideCredentials; +#[cfg(feature = "parquet")] use axum::async_trait; +#[cfg(feature = "parquet")] use object_store::aws::AwsCredential; #[cfg(feature = "parquet")] use object_store::azure::MicrosoftAzureBuilder; @@ -10,9 +13,12 @@ use object_store::ObjectStore; #[cfg(feature = "parquet")] use object_store::{aws::AmazonS3Builder, ClientOptions}; use serde::{Deserialize, Serialize}; +#[cfg(feature = "parquet")] use std::sync::Arc; +#[cfg(feature = "parquet")] use tokio::sync::RwLock; +#[cfg(feature = "parquet")] lazy_static::lazy_static! { pub static ref S3_CACHE_SETTINGS: Arc>>> = Arc::new(RwLock::new(None)); @@ -178,6 +184,7 @@ use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region} #[cfg(feature = "parquet")] use object_store::CredentialProvider; +#[cfg(feature = "parquet")] pub fn build_s3_client(s3_resource_ref: &S3Resource, credential_providers: Option) -> error::Result> { let s3_resource = s3_resource_ref.clone(); @@ -315,6 +322,7 @@ pub struct S3Settings { pub store_logs: Option, } +#[cfg(feature = "parquet")] fn none_if_empty(s: Option) -> Option { if s.is_none() || s.as_ref().unwrap().is_empty() { None @@ -352,11 +360,13 @@ pub async fn build_s3_client_from_settings(settings: S3Settings) -> error::Resul build_s3_client(&s3_resource, credentials_provider) } +#[cfg(feature = "parquet")] #[derive(Debug)] struct AwsCredentialAdapter { pub inner: DefaultCredentialsChain, } +#[cfg(feature = "parquet")] #[async_trait] impl CredentialProvider for AwsCredentialAdapter { type Credential = AwsCredential; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 5d5308430d..afa01a44f0 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -791,6 +791,7 @@ pub async fn handle_python_reqs( } } + #[cfg(all(feature = "enterprise", feature = "parquet"))] enum PullFromTar { Pulled(String), NotPulled(String, String),