diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1f02308fbb..eaf6ef8e37 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1096,6 +1096,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "aws-smithy-types-convert" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df786cc1aea35d24b609f7a32d05570916edfe7b3e09e81f2faf365f9062f647" +dependencies = [ + "aws-smithy-types", + "chrono", +] + [[package]] name = "aws-smithy-xml" version = "0.60.9" @@ -14650,6 +14660,7 @@ dependencies = [ "async-stream", "aws-config", "aws-sdk-sts", + "aws-smithy-types-convert", "axum", "backon", "bytes", @@ -14674,6 +14685,7 @@ dependencies = [ "magic-crypt", "mail-send", "object_store", + "openidconnect", "opentelemetry", "opentelemetry-appender-tracing", "opentelemetry-otlp", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 88aa2ffab6..069aca4367 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -59,7 +59,7 @@ embedding = ["windmill-api/embedding"] parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"] flow_testing = ["windmill-worker/flow_testing"] -openidconnect = ["windmill-api/openidconnect"] +openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] @@ -346,7 +346,7 @@ openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" aws-sdk-sqs = "1.57.0" aws-sdk-sts = "^1" - +aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] } crc = "^3" tar = "^0" http = "^1" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9b39544645..04b305d5a2 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -72e6260ca886628cf1ba271bc058e6ecfdecdae5 \ No newline at end of file +6899b8151329218a1df59964dac57e0e004ae25a \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 8c451f63a0..a4148d93c6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -69,7 +69,7 @@ use tikv_jemallocator::Jemalloc; static GLOBAL: Jemalloc = Jemalloc; #[cfg(feature = "parquet")] -use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; +use windmill_common::global_settings::OBJECT_STORE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, @@ -92,7 +92,7 @@ use crate::monitor::{ }; #[cfg(feature = "parquet")] -use crate::monitor::reload_s3_cache_setting; +use windmill_common::s3_helpers::reload_object_store_setting; const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; @@ -907,9 +907,9 @@ Windmill Community Edition {GIT_VERSION} reload_job_default_timeout_setting(&conn).await }, #[cfg(feature = "parquet")] - OBJECT_STORE_CACHE_CONFIG_SETTING => { + OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { - reload_s3_cache_setting(&db).await + reload_object_store_setting(&db).await; } }, SCIM_TOKEN_SETTING => { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 5c4d166beb..d2a5657c6c 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -33,8 +33,11 @@ use windmill_common::ee::low_disk_alerts; #[cfg(feature = "enterprise")] use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts}; +use windmill_common::client::AuthedClient; #[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; +#[cfg(feature = "parquet")] +use windmill_common::s3_helpers::reload_object_store_setting; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, @@ -75,19 +78,13 @@ use windmill_common::{ }; use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload}; use windmill_worker::{ - handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, + handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, }; #[cfg(feature = "parquet")] -use windmill_common::s3_helpers::{ - build_object_store_from_settings, build_s3_client_from_settings, S3Settings, - OBJECT_STORE_CACHE_SETTINGS, -}; - -#[cfg(feature = "parquet")] -use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; +use windmill_common::s3_helpers::ObjectStoreReload; #[cfg(feature = "enterprise")] use crate::ee::verify_license_key; @@ -241,7 +238,23 @@ pub async fn initial_load( #[cfg(feature = "parquet")] if !disable_s3_store { if let Some(db) = conn.as_sql() { - reload_s3_cache_setting(db).await; + let db2 = db.clone(); + match reload_object_store_setting(db).await { + ObjectStoreReload::Later => { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + match reload_object_store_setting(&db2).await { + ObjectStoreReload::Later => { + tracing::error!("Giving up on loading object store setting"); + } + ObjectStoreReload::Never => { + tracing::info!("Object store setting successfully loaded"); + } + } + }); + } + ObjectStoreReload::Never => (), + } } } @@ -631,7 +644,7 @@ async fn send_log_file_to_object_store( } #[cfg(feature = "parquet")] - let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone(); + let s3_client = windmill_common::s3_helpers::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE) @@ -917,10 +930,7 @@ async fn delete_log_files_from_disk_and_store( _s3_prefix: &str, ) { #[cfg(feature = "parquet")] - let os = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone(); + let os = windmill_common::s3_helpers::get_object_store().await; #[cfg(not(feature = "parquet"))] let os: Option<()> = None; @@ -1101,64 +1111,6 @@ pub async fn reload_delete_logs_periodically_setting(conn: &Connection) { } } -#[cfg(feature = "parquet")] -pub async fn reload_s3_cache_setting(db: &DB) { - use windmill_common::{ - ee::{get_license_plan, LicensePlan}, - s3_helpers::ObjectSettings, - }; - - let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CACHE_CONFIG_SETTING).await; - if let Err(e) = s3_config { - tracing::error!("Error reloading s3 cache config: {:?}", e) - } else { - if let Some(v) = s3_config.unwrap() { - if matches!(get_license_plan().await, LicensePlan::Pro) { - tracing::error!("S3 cache is not available for pro plan"); - return; - } - let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await; - let setting = serde_json::from_value::(v); - if let Err(e) = setting { - tracing::error!("Error parsing s3 cache config: {:?}", e) - } else { - let setting = setting.unwrap(); - let bucket = setting.get_bucket().map(|b| b.to_string()); - let s3_client = build_object_store_from_settings(setting).await; - if let Err(e) = s3_client { - tracing::error!("Error building s3 client from settings: {:?}", e) - } else { - tracing::info!("Loaded object store {:?}", bucket); - *s3_cache_settings = Some(s3_client.unwrap()); - } - } - } else { - let mut s3_cache_settings = OBJECT_STORE_CACHE_SETTINGS.write().await; - if std::env::var("S3_CACHE_BUCKET").is_ok() { - if matches!(get_license_plan().await, LicensePlan::Pro) { - tracing::error!("S3 cache is not available for pro plan"); - return; - } - *s3_cache_settings = build_s3_client_from_settings(S3Settings { - bucket: None, - region: None, - access_key: None, - secret_key: None, - endpoint: None, - store_logs: None, - path_style: None, - allow_http: None, - port: None, - }) - .await - .ok(); - } else { - *s3_cache_settings = None; - } - } - } -} - pub async fn reload_job_default_timeout_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 276377b1ca..9f6de0c5ca 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -18,7 +18,7 @@ benchmark = [] embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] -openidconnect = ["dep:openidconnect"] +openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] tantivy = ["dep:windmill-indexer"] kafka = ["dep:rdkafka"] nats = ["dep:async-nats", "dep:nkeys"] diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 5c2041fd85..cf9e7d06de 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -83,8 +83,6 @@ use windmill_common::{ }, }; -#[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; #[cfg(feature = "prometheus")] use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; @@ -1058,7 +1056,7 @@ async fn get_logs_from_store( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { tracing::debug!("Getting logs from store: {file_index:?}"); - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { tracing::debug!("object store client present, streaming from there"); let logs = logs.to_string(); @@ -4962,10 +4960,7 @@ async fn run_bundle_preview_script( uploaded = true; #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone(); + let object_store = windmill_common::s3_helpers::get_object_store().await; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let object_store: Option<()> = None; @@ -5663,7 +5658,7 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { let file = os .get(&object_store::path::Path::from(format!("logs/{file_p}"))) .await; diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 90da2187dc..a827fa5079 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -410,10 +410,7 @@ async fn create_snapshot_script( uploaded = true; #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone(); + let object_store = windmill_common::s3_helpers::get_object_store().await; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let object_store: Option<()> = None; @@ -1327,10 +1324,12 @@ async fn raw_script_by_path_internal( w_id ) .fetch_one(&db) - .await?; - if exists.unwrap_or(false) { + .await? + .unwrap_or(false); + + if exists { return Err(Error::NotFound(format!( - "Script {path} not visible to {} but exists", + "Script {path} exists but {} does not have permissions to access it", authed.username ))); } diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index b11646fbe0..0b3ca56af7 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -98,10 +98,7 @@ async fn get_log_file( require_devops_role(&db, &email).await?; let path = path.to_path(); #[cfg(feature = "parquet")] - let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone(); + let s3_client = windmill_common::s3_helpers::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index a270074cc0..b589713b91 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -120,12 +120,15 @@ use windmill_common::s3_helpers::build_object_store_from_settings; #[cfg(feature = "parquet")] pub async fn test_s3_bucket( _authed: ApiAuthed, + Extension(db): Extension, Json(test_s3_bucket): Json, ) -> error::Result { use bytes::Bytes; use futures::StreamExt; - let client = build_object_store_from_settings(test_s3_bucket).await?; + let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) + .await? + .store; let mut list = client.list(Some(&object_store::path::Path::from("".to_string()))); let first_file = list.next().await; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 7eefe5b2cc..17764b1253 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -12,14 +12,14 @@ tantivy = [] prometheus = ["dep:prometheus"] loki = ["dep:tracing-loki"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:datafusion"] +parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"] aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"] smtp = ["dep:mail-send"] scoped_cache = [] cloud = [] - +openidconnect = ["dep:openidconnect"] [lib] name = "windmill_common" path = "src/lib.rs" @@ -62,6 +62,7 @@ object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } +aws-smithy-types-convert = { workspace = true, optional = true } indexmap.workspace = true bytes.workspace = true mail-send = { workspace = true, optional = true } @@ -75,6 +76,7 @@ windmill-parser-ts.workspace = true windmill-parser-py.workspace = true jsonwebtoken.workspace = true backon.workspace = true +openidconnect = { workspace = true, optional = true } strum.workspace = true strum_macros.workspace = true diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs new file mode 100644 index 0000000000..95d9f64bd2 --- /dev/null +++ b/backend/windmill-common/src/client.rs @@ -0,0 +1,200 @@ +use anyhow::Context; +use reqwest::{Body, Response}; +use serde::de::DeserializeOwned; + +use crate::utils::HTTP_CLIENT; + +#[derive(Clone)] +pub struct AuthedClient { + pub base_internal_url: String, + pub workspace: String, + pub token: String, + pub force_client: Option, +} + +impl AuthedClient { + pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result { + self.force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .get(url) + .query(&query) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?, + ) + .send() + .await + .map_err(|e| { + tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}"); + anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}") + }) + } + + pub async fn get_id_token(&self, audience: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/oidc/token/{}", + self.base_internal_url, self.workspace, audience + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding oidc token as json string")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_resource_value(&self, path: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/resources/get_value/{}", + self.base_internal_url, self.workspace, path + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding resource value as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_variable_value(&self, path: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/variables/get_value/{}", + self.base_internal_url, self.workspace, path + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding variable value as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_resource_value_interpolated( + &self, + path: &str, + job_id: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/resources/get_value_interpolated/{}", + self.base_internal_url, self.workspace, path + ); + let mut query = Vec::with_capacity(1usize); + if let Some(v) = &job_id { + query.push(("job_id", v.to_string())); + } + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding interpolated resource value as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_completed_job_result( + &self, + path: &str, + json_path: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/jobs_u/completed/get_result/{}", + self.base_internal_url, self.workspace, path + ); + let query = if let Some(json_path) = json_path { + vec![("json_path", json_path)] + } else { + vec![] + }; + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding completed job result as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn get_result_by_id( + &self, + flow_job_id: &str, + node_id: &str, + json_path: Option, + ) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/jobs/result_by_id/{}/{}", + self.base_internal_url, self.workspace, flow_job_id, node_id + ); + let query = if let Some(json_path) = json_path { + vec![("json_path", json_path)] + } else { + vec![] + }; + let response = self.get(&url, query).await?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding result by id as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + + pub async fn upload_s3_file( + &self, + workspace_id: &str, + object_key: String, + storage: Option, + body: S, + ) -> anyhow::Result<()> + where + S: futures::stream::TryStream + Send + 'static, + S::Error: Into>, + bytes::Bytes: From, + { + let mut query = vec![("file_key", object_key)]; + if let Some(storage) = storage { + query.push(("storage", storage)); + } + let response = self + .force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .post(format!( + "{}/api/w/{}/job_helpers/upload_s3_file", + self.base_internal_url, workspace_id + )) + .query(&query) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) + .map_err(|e| anyhow::anyhow!(e.to_string()))?, + ) + .body(Body::wrap_stream(body)) + .send() + .await + .context(format!("Sent upload_s3_file request",)) + .map_err(|e| anyhow::anyhow!(e.to_string()))?; + + match response.status().as_u16() { + 200u16 => Ok(()), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, + } + } +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 61184b627c..895384e855 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -30,7 +30,7 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics"; pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; -pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config"; +pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; diff --git a/backend/windmill-common/src/job_s3_helpers_ee.rs b/backend/windmill-common/src/job_s3_helpers_ee.rs index b00c00a583..d5d6ac4052 100644 --- a/backend/windmill-common/src/job_s3_helpers_ee.rs +++ b/backend/windmill-common/src/job_s3_helpers_ee.rs @@ -1,18 +1,34 @@ -use std::future::Future; +use crate::s3_helpers::{ObjectStoreResource, StorageResourceType}; -use crate::{ - error::Error, - s3_helpers::{ObjectStoreResource, StorageResourceType}, -}; - -pub async fn get_s3_resource_internal<'c, F, Fut>( +pub async fn get_s3_resource_internal<'c>( _resource_type: StorageResourceType, _s3_resource_value_raw: serde_json::Value, - _gen_token: F, -) -> crate::error::Result -where - F: FnOnce(String) -> Fut, - Fut: Future> + Send + 'static, -{ + _gen_token: TokenGenerator<'c>, + _db: &crate::DB, +) -> crate::error::Result { + todo!() +} + +pub enum TokenGenerator<'c> { + AsClient(&'c crate::client::AuthedClient), + AsServerInstance(), +} + +impl<'c> TokenGenerator<'c> { + pub async fn gen_token( + &self, + _audience: &str, + _db: Option<&crate::DB>, + ) -> anyhow::Result { + todo!() + } +} + +#[cfg(feature = "parquet")] +pub(crate) async fn generate_s3_aws_oidc_resource<'c>( + _clone: crate::s3_helpers::S3AwsOidcResource, + _token_generator: TokenGenerator<'c>, + _init_private_key: Option<&sqlx::Pool>, +) -> crate::error::Result { todo!() } diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 8966b59c72..1e28d373be 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -608,11 +608,11 @@ pub async fn get_logs_from_store( logs: &str, log_file_index: &Option>, ) -> Option>> { - use crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; + use crate::s3_helpers::get_object_store; if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + if let Some(os) = get_object_store().await { let logs = logs.to_string(); let stream = async_stream::stream! { for file_p in file_index.clone() { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index bc7f244787..6d92cd8f55 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -30,6 +30,7 @@ pub mod auth; #[cfg(feature = "benchmark")] pub mod bench; pub mod cache; +pub mod client; pub mod db; pub mod ee; pub mod email_ee; @@ -43,6 +44,9 @@ pub mod job_metrics; #[cfg(feature = "parquet")] pub mod job_s3_helpers_ee; +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +pub mod oidc_ee; + pub mod jobs; pub mod jwt; pub mod more_serde; diff --git a/backend/windmill-common/src/oidc_ee.rs b/backend/windmill-common/src/oidc_ee.rs new file mode 100644 index 0000000000..e7a157b04d --- /dev/null +++ b/backend/windmill-common/src/oidc_ee.rs @@ -0,0 +1,198 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2023 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +use { + crate::db::DB, + crate::{auth::IdToken as WindmillIdToken, error::Result}, + anyhow, + openidconnect::{ + core::{CoreJwsSigningAlgorithm, CoreRsaPrivateSigningKey}, + IssuerUrl, JsonWebKeyId, + }, + std::process::Command, +}; + +#[cfg(feature = "openidconnect")] +use openidconnect::AdditionalClaims; + +#[cfg(feature = "openidconnect")] +impl AdditionalClaims for JobClaim {} + +#[cfg(feature = "openidconnect")] +impl AdditionalClaims for WorkspaceClaim {} + +#[cfg(feature = "openidconnect")] +impl AdditionalClaims for InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct WorkspaceClaim { + pub workspace: String, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct JobClaim { + pub job_id: String, + pub path: Option, + pub flow_path: Option, + pub groups: Vec, + pub username: String, + pub email: String, + pub workspace: String, +} + +lazy_static::lazy_static! { + static ref PRIVATE_KEY: RwLock> = RwLock::new(None); +} + +pub async fn generate_id_token( + db: Option<&DB>, + claim: T, + audience: &str, + identifier: String, + email: Option, +) -> Result { + use chrono::{Duration, Utc}; + use openidconnect::{ + core::{CoreGenderClaim, CoreJweContentEncryptionAlgorithm}, + Audience, EndUserEmail, IdToken, IdTokenClaims, StandardClaims, SubjectIdentifier, + }; + + let private_key = get_private_key(db).await?; + + let issue_url = format!("{}/api/oidc/", crate::BASE_URL.read().await.clone()); + let issue_time = Utc::now(); + let expiration = issue_time + Duration::try_hours(48).unwrap(); + let id_token = IdToken::< + T, + CoreGenderClaim, + CoreJweContentEncryptionAlgorithm, + CoreJwsSigningAlgorithm, + >::new( + IdTokenClaims::::new( + // Specify the issuer URL for the OpenID Connect Provider. + IssuerUrl::new(issue_url) + .map_err(|e| anyhow::anyhow!("Failed to generate IssueUrl: {}", e))?, + // The audience is usually a single entry with the client ID of the client for whom + // the ID token is intended. This is a required claim. + vec![Audience::new(audience.to_string())], + // The ID token expiration is usually much shorter than that of the access or refresh + // tokens issued to clients. + expiration, + // The issue time is usually the current time. + issue_time, + // Set the standard claims defined by the OpenID Connect Core spec. + StandardClaims::new( + // Stable subject identifiers are recommended in place of e-mail addresses or other + // potentially unstable identifiers. This is the only required claim. + SubjectIdentifier::new(identifier), + ) + // Optional: specify the user's e-mail address. This should only be provided if the + // client has been granted the 'profile' or 'email' scopes. + .set_email(email.map(|x| EndUserEmail::new(x))) + // Optional: specify whether the provider has verified the user's e-mail address. + .set_email_verified(Some(true)), + // OpenID Connect Providers may supply custom claims by providing a struct that + // implements the AdditionalClaims trait. This requires manually using the + // generic IdTokenClaims struct rather than the CoreIdTokenClaims type alias, + // however. + claim, + ), + // The private key used for signing the ID token. For confidential clients (those able + // to maintain a client secret), a CoreHmacKey can also be used, in conjunction + // with one of the CoreJwsSigningAlgorithm::HmacSha* signing algorithms. When using an + // HMAC-based signing algorithm, the UTF-8 representation of the client secret should + // be used as the HMAC key. + &CoreRsaPrivateSigningKey::from_pem( + &private_key, + Some(JsonWebKeyId::new("windmill".to_string())), + ) + .map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?, + // Uses the RS256 signature algorithm. This crate supports any RS*, PS*, or HS* + // signature algorithm. + CoreJwsSigningAlgorithm::RsaSsaPkcs1V15Sha256, + // When returning the ID token alongside an access token (e.g., in the Authorization Code + // flow), it is recommended to pass the access token here to set the `at_hash` claim + // automatically. + None, + // When returning the ID token alongside an authorization code (e.g., in the implicit + // flow), it is recommended to pass the authorization code here to set the `c_hash` claim + // automatically. + None, + ) + .map_err(|e| anyhow::anyhow!("Failed to generate token: {}", e))?; + + Ok(WindmillIdToken::new(id_token.to_string(), expiration)) +} + +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result { + if let Some(key) = PRIVATE_KEY.read().await.clone() { + return Ok(key); + } else if let Some(db) = db { + let key = sqlx::query_scalar!( + "SELECT value->>'private_key' FROM global_settings WHERE name = 'rsa_keys'", + ) + .fetch_optional(db) + .await? + .flatten(); + + let key = key.filter(|s| !s.is_empty()); + + if let Some(key) = key { + return Ok(key); + } else { + let keys = gen_pems(db).await?; + return Ok(keys.private_key); + } + } else { + return Err(anyhow::anyhow!("Private key not found and no db provided")); + } +} + +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +#[derive(Debug, Clone, serde::Serialize)] +struct Keys { + private_key: String, +} + +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +async fn gen_pems(db: &DB) -> anyhow::Result { + use anyhow::anyhow; + + let private_key_cmd = Command::new("openssl") + .arg("genrsa") + .arg("--traditional") + .arg("2048") + .output() + .expect("failed to execute process"); + + let private_key = String::from_utf8(private_key_cmd.stdout)?; + + tracing::debug!("Generated private key: {}", private_key); + + if private_key.is_empty() { + return Err(anyhow!("Failed to generate RSA key: key is empty")); + } + + let keys = Keys { private_key }; + + sqlx::query!( + r#"INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)"#, + serde_json::to_value(&keys).unwrap() + ) + .execute(db) + .await?; + + Ok(keys) +} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 698fad1c4d..29f59cf07c 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -4,6 +4,7 @@ use crate::error; use aws_sdk_sts::config::ProvideCredentials; #[cfg(feature = "parquet")] use axum::async_trait; +use chrono::{DateTime, Utc}; #[cfg(feature = "parquet")] use object_store::aws::AwsCredential; #[cfg(feature = "parquet")] @@ -17,6 +18,7 @@ use reqwest::header::HeaderMap; use serde::{Deserialize, Serialize}; #[cfg(feature = "parquet")] use std::sync::{Arc, Mutex}; + #[cfg(feature = "parquet")] use tokio::sync::RwLock; @@ -46,9 +48,170 @@ use tokio::task; use windmill_parser_sql::S3ModeFormat; #[cfg(feature = "parquet")] -lazy_static::lazy_static! { +#[derive(Clone)] +pub struct ExpirableObjectStore { + pub store: Arc, + pub refresh: Option, +} - pub static ref OBJECT_STORE_CACHE_SETTINGS: Arc>>> = Arc::new(RwLock::new(None)); +#[cfg(feature = "parquet")] +#[derive(Clone)] +pub struct ObjectStoreRefresh { + refresh: Option>, + settings: ObjectSettings, +} + +#[cfg(feature = "parquet")] +impl ObjectStoreRefresh { + pub fn new(settings: ObjectSettings, refresh: Option>) -> Self { + Self { settings, refresh } + } + fn refresh_needed(&self) -> bool { + if let Some(refresh) = self.refresh { + if refresh < Utc::now() - chrono::Duration::minutes(1) { + return true; + } + } + return false; + } + + async fn refresh(&self) -> Option { + return build_object_store_from_settings(self.settings.clone(), None) + .await + .map_err(|e| { + tracing::error!("Error building s3 client from settings: {:?}", e); + e + }) + .ok(); + } +} + +#[cfg(feature = "parquet")] +impl From> for ExpirableObjectStore { + fn from(store: Arc) -> Self { + Self { store, refresh: None } + } +} + +// #[cfg(feature = "parquet")] + +// impl ExpirableObjectStore { +// pub fn new(store: Arc, expiration: Option>) -> Self { +// Self { store, expiration } +// } +// } + +#[cfg(feature = "parquet")] +lazy_static::lazy_static! { + pub static ref OBJECT_STORE_SETTINGS: Arc>> = Arc::new(RwLock::new(None)); +} + +#[cfg(feature = "parquet")] +pub async fn get_object_store() -> Option> { + let settings = OBJECT_STORE_SETTINGS.read().await; + if let Some(s) = settings.as_ref() { + match &s.refresh { + Some(refresh) => { + if refresh.refresh_needed() { + let refresh = refresh.clone(); + drop(settings); + let new_store = refresh.refresh().await; + if let Some(new_store) = new_store { + let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; + let arc = new_store.store.clone(); + *s3_cache_settings = Some(new_store); + return Some(arc); + } else { + return None; + } + } else { + return Some(s.store.clone()); + } + } + None => { + return Some(s.store.clone()); + } + } + } else { + return None; + } +} + +#[cfg(feature = "parquet")] +pub enum ObjectStoreReload { + //if the jwks endpoints are not up yet, we should retry later soon + Later, + Never, +} + +#[cfg(feature = "parquet")] +pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { + use crate::{ + ee::{get_license_plan, LicensePlan}, + global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING}, + s3_helpers::ObjectSettings, + }; + + let s3_config = load_value_from_global_settings(db, OBJECT_STORE_CONFIG_SETTING).await; + if let Err(e) = s3_config { + tracing::error!("Error reloading s3 cache config: {:?}", e) + } else { + if let Some(v) = s3_config.unwrap() { + if matches!(get_license_plan().await, LicensePlan::Pro) { + tracing::error!("S3 cache is not available for pro plan"); + return ObjectStoreReload::Never; + } + let setting = serde_json::from_value::(v); + match setting { + Ok(setting) => { + let is_oidc = matches!(setting, ObjectSettings::AwsOidc(_)); + let s3_client = build_object_store_from_settings(setting, Some(db)).await; + match s3_client { + Ok(s3_client) => { + let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; + *s3_cache_settings = Some(s3_client); + } + Err(e) => { + if is_oidc { + tracing::error!("Error building s3 client from oidc settings. It may be due to the jwks endpoints not being up yet, it will be attempted again in 10s to leave time for the server to be ready: {:?}", e); + return ObjectStoreReload::Later; + } else { + tracing::error!("Error building s3 client from settings: {:?}", e); + } + } + } + } + Err(e) => { + tracing::error!("Error parsing s3 cache config: {:?}", e) + } + } + } else { + let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; + if std::env::var("S3_CACHE_BUCKET").is_ok() { + if matches!(get_license_plan().await, LicensePlan::Pro) { + tracing::error!("S3 cache is not available for pro plan"); + return ObjectStoreReload::Never; + } + *s3_cache_settings = build_s3_client_from_settings(S3Settings { + bucket: None, + region: None, + access_key: None, + secret_key: None, + endpoint: None, + store_logs: None, + path_style: None, + allow_http: None, + port: None, + }) + .await + .ok() + .map(|x| ExpirableObjectStore::from(x)) + } else { + *s3_cache_settings = None; + } + } + } + return ObjectStoreReload::Never; } #[derive(Serialize, Deserialize, Debug)] @@ -81,6 +244,15 @@ pub enum ObjectStoreResource { Azure(AzureBlobResource), } +impl ObjectStoreResource { + pub fn expiration(&self) -> Option> { + match self { + ObjectStoreResource::S3(s3_resource) => s3_resource.expiration, + _ => None, + } + } +} + #[derive(Deserialize, Debug)] pub enum StorageResourceType { S3, @@ -104,6 +276,8 @@ pub struct S3Resource { #[serde(rename = "pathStyle")] pub path_style: Option, pub token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expiration: Option>, pub port: Option, } @@ -126,7 +300,7 @@ pub struct AzureBlobResource { pub federated_token_file: Option, } -#[derive(Debug, Deserialize, Serialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone, Hash)] pub struct S3AwsOidcResource { #[serde(rename = "bucket")] pub bucket: String, @@ -412,6 +586,7 @@ pub enum ObjectStoreSettings { pub enum ObjectSettings { S3(S3Settings), Azure(AzureBlobResource), + AwsOidc(S3AwsOidcResource), } impl ObjectSettings { @@ -419,6 +594,7 @@ impl ObjectSettings { match self { ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(), ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name), + ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket), } } } @@ -426,12 +602,31 @@ impl ObjectSettings { #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, -) -> error::Result> { + init_private_key: Option<&crate::DB>, +) -> error::Result { match settings { - ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings).await, + ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) + .await + .map(|x| ExpirableObjectStore::from(x)), ObjectSettings::Azure(azure_settings) => { let azure_blob_resource = azure_settings; - build_azure_blob_client(&azure_blob_resource) + build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) + } + ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { + let token_generator = crate::job_s3_helpers_ee::TokenGenerator::AsServerInstance(); + let res = crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource( + s3_aws_oidc_settings.clone(), + token_generator, + init_private_key, + ) + .await?; + + build_object_store_client(&res) + .await + .map(|x| ExpirableObjectStore { + store: x, + refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), + }) } } } @@ -479,6 +674,7 @@ pub async fn build_s3_client_from_settings( path_style: settings.path_style, port: settings.port, token: None, + expiration: None, }; build_s3_client(&s3_resource).await diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 2d1e295432..aa1a715758 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -582,11 +582,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo (true, format!("loaded from local cache: {}\n", bin_path)) } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = crate::s3_helpers::get_object_store().await { let started = std::time::Instant::now(); use crate::s3_helpers::attempt_fetch_bytes; @@ -629,11 +625,7 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { return true; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = crate::s3_helpers::get_object_store().await { return os .get(&object_store::path::Path::from(_remote_path)) .await @@ -651,11 +643,7 @@ pub async fn save_cache( ) -> crate::error::Result { let mut _cached_to_s3 = false; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = crate::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = crate::s3_helpers::get_object_store().await { use object_store::path::Path; let file_to_cache = if is_dir { let tar_path = format!( diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index ea94b2c1fb..1783641da3 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -31,9 +31,10 @@ use crate::{ }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - AuthedClient, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, - PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + PY_INSTALL_DIR, TZ_ENV, }; +use windmill_common::client::AuthedClient; lazy_static::lazy_static! { static ref ANSIBLE_PLAYBOOK_PATH: String = @@ -1190,7 +1191,7 @@ async fn create_file_resources( job_dir: &str, args: Option<&HashMap>>, r: &AnsibleRequirements, - client: &crate::AuthedClient, + client: &AuthedClient, conn: &Connection, ) -> error::Result> { let mut logs = String::new(); @@ -1267,7 +1268,7 @@ async fn create_file_resources( } async fn get_resource_or_variable_content( - client: &crate::AuthedClient, + client: &AuthedClient, path: &ResourceOrVariablePath, job_id: String, ) -> anyhow::Result { diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 1887873364..ec08003dd3 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -43,9 +43,11 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, }; +use windmill_common::client::AuthedClient; + #[cfg(windows)] use crate::SYSTEM_ROOT; diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 3f2f63df5a..a93a067b02 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -4,6 +4,7 @@ use futures::future::BoxFuture; use futures::{FutureExt, StreamExt}; use reqwest::Client; use serde_json::{json, value::RawValue, Value}; +use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; use windmill_common::s3_helpers::convert_json_line_stream; use windmill_common::worker::Connection; @@ -16,15 +17,12 @@ use windmill_queue::CanceledBy; use serde::Deserialize; +use crate::common::{build_args_values, resolve_job_timeout}; use crate::common::{ build_http_client, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData, }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; -use crate::{ - common::{build_args_values, resolve_job_timeout}, - AuthedClient, -}; use gcp_auth::{AuthenticationManager, CustomServiceAccount}; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 6a6ad120e7..fa64832ee4 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -20,10 +20,11 @@ use crate::{ read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, - NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, + BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, + DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, + PATH_ENV, PROXY_ENVS, TZ_ENV, }; +use windmill_common::client::AuthedClient; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -612,10 +613,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - let object_store = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone(); + let object_store = windmill_common::s3_helpers::get_object_store().await; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let object_store: Option<()> = None; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bec5f4c7ef..9955256d13 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -44,10 +44,8 @@ use windmill_common::{variables, DB}; use tokio::{io::AsyncWriteExt, process::Child, time::Instant}; use crate::agent_workers::UPDATE_PING_URL; -use crate::{ - AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, - PATH_ENV, -}; +use crate::{DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV}; +use windmill_common::client::AuthedClient; pub async fn build_args_map<'a>( job: &'a MiniPulledJob, @@ -782,19 +780,17 @@ async fn get_workspace_s3_resource_path( } }; - let client2 = client.clone(); - let token_fn = |audience: String| async move { - client2 - .get_id_token(&audience) - .await - .map_err(|e| windmill_common::error::Error::from(e)) - }; let s3_resource_value_raw = client .get_resource_value::(path.as_str()) .await?; - get_s3_resource_internal(rt, s3_resource_value_raw, token_fn) - .await - .map(Some) + get_s3_resource_internal( + rt, + s3_resource_value_raw, + windmill_common::job_s3_helpers_ee::TokenGenerator::AsClient(client), + db, + ) + .await + .map(Some) } #[cfg(feature = "parquet")] @@ -1109,7 +1105,7 @@ pub async fn par_install_language_dependencies<'a>( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + if windmill_common::s3_helpers::OBJECT_STORE_SETTINGS .read() .await .is_none() @@ -1264,11 +1260,7 @@ pub async fn par_install_language_dependencies<'a>( #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { Some(crate::global_cache::pull_from_tar( os, path.clone(), @@ -1449,11 +1441,7 @@ pub async fn par_install_language_dependencies<'a>( }; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( os, @@ -1541,11 +1529,7 @@ pub async fn par_install_language_dependencies<'a>( }; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS - .read() - .await - .clone() - { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { let language_name = language_name.to_owned(); tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( @@ -1591,7 +1575,7 @@ pub struct S3ModeWorkerData { } impl S3ModeWorkerData { - pub async fn upload(&self, stream: S) -> error::Result<()> + pub async fn upload(&self, stream: S) -> anyhow::Result<()> where S: futures::stream::TryStream + Send + 'static, S::Error: Into>, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 2054e279db..a8ed06a2e2 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -36,7 +36,7 @@ use crate::{ }; use crate::common::OccupancyMetrics; -use crate::AuthedClient; +use windmill_common::client::AuthedClient; #[cfg(windows)] use crate::SYSTEM_ROOT; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 696957f463..2215c94d8d 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -11,9 +11,11 @@ use crate::{ start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, + DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; +use windmill_common::client::AuthedClient; + use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, worker::write_file, BASE_URL}; use windmill_common::{ diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 13a85cb55b..2217c775ba 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -19,9 +19,10 @@ use crate::{ start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, - GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, + NSJAIL_PATH, PATH_ENV, TZ_ENV, }; +use windmill_common::client::AuthedClient; const GO_REQ_SPLITTER: &str = "//go.sum\n"; const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto"); diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index 1fbc547c30..d9117adb53 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -12,7 +12,8 @@ use serde::Deserialize; use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; -use crate::{common::build_args_map, AuthedClient}; +use crate::common::build_args_map; +use windmill_common::client::AuthedClient; #[derive(Deserialize)] struct GraphqlApi { diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 65f157ce79..f62de150a9 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -24,9 +24,11 @@ use crate::{ create_args_and_out_file, get_reserved_variables, par_install_language_dependencies, read_result, start_child_process, OccupancyMetrics, RequiredDependency, }, - handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, + handle_child, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; +use windmill_common::client::AuthedClient; + lazy_static::lazy_static! { static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string()); diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 5c9af6760b..77ccd9b15f 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -48,7 +48,8 @@ use windmill_common::worker::{write_file, TMP_DIR}; use windmill_common::flow_status::JobResult; use windmill_queue::CanceledBy; -use crate::{common::OccupancyMetrics, AuthedClient}; +use crate::common::OccupancyMetrics; +use windmill_common::client::AuthedClient; #[cfg(feature = "deno_core")] use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller}; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index ca6b4785c7..7f5ec7c116 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -22,7 +22,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; -use crate::AuthedClient; +use windmill_common::client::AuthedClient; use serde::Deserializer; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index dae89112fe..ffd84db175 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -13,6 +13,7 @@ use serde_json::{json, value::RawValue, Value}; use std::str::FromStr; use tokio::sync::Mutex; use windmill_common::{ + client::AuthedClient, error::{to_anyhow, Error}, s3_helpers::convert_json_line_stream, worker::{to_raw_value, Connection}, @@ -28,7 +29,6 @@ use crate::{ common::{build_args_values, s3_mode_args_to_worker_data, OccupancyMetrics, S3ModeWorkerData}, handle_child::run_future_with_polling_update_job_poller, sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, - AuthedClient, }; #[derive(Deserialize)] diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index b2f6590c41..c31d979c27 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,8 +16,10 @@ use crate::{ create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, - handle_child, AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; +use windmill_common::client::AuthedClient; + const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto"); lazy_static::lazy_static! { diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index 2147a53e45..244cbb1958 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -27,9 +27,9 @@ use crate::{ OccupancyMetrics, S3ModeWorkerData, }, handle_child::run_future_with_polling_update_job_poller, - sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, - AuthedClient, + sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args }; +use windmill_common::client::AuthedClient; #[derive(Deserialize)] struct OracleDatabase { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index fdd9a3d7a4..67c2d945aa 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -41,11 +41,11 @@ use crate::common::{ }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; -use crate::{AuthedClient, MAX_RESULT_SIZE}; +use crate::MAX_RESULT_SIZE; use bytes::Buf; use lazy_static::lazy_static; use urlencoding::encode; - +use windmill_common::client::AuthedClient; #[derive(Deserialize)] struct PgDatabase { host: String, diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index ac50beb99f..ec8478a9a5 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -20,9 +20,10 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, + COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; +use windmill_common::client::AuthedClient; const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto"); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 0289ef1560..76ee967656 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -70,7 +70,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); use crate::global_cache::{build_tar_and_push, pull_from_tar}; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] -use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; +use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ @@ -79,9 +79,10 @@ use crate::{ }, handle_child::handle_child, worker_utils::ping_job_status, - AuthedClient, PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; +use windmill_common::client::AuthedClient; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -1424,7 +1425,7 @@ pub async fn handle_python_reqs( } #[cfg(all(feature = "enterprise", feature = "parquet", unix))] - if OBJECT_STORE_CACHE_SETTINGS.read().await.is_none() { + if OBJECT_STORE_SETTINGS.read().await.is_none() { (s3_pull, s3_push) = (false, false); } @@ -1735,7 +1736,7 @@ pub async fn handle_python_reqs( let start = std::time::Instant::now(); #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if is_not_pro { - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), @@ -1889,7 +1890,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { - if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { + if let Some(os) = windmill_common::s3_helpers::get_object_store().await { tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false)); } } diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 2ca1539b85..342c53a301 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -29,8 +29,7 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, - WrappedError, + append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError, }; use serde_json::{json, value::RawValue}; @@ -44,9 +43,10 @@ use crate::{ common::{error_to_value, read_result, save_in_cache}, otel_ee::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, - AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, - UpdateFlow, INIT_SCRIPT_TAG, + JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, UpdateFlow, + INIT_SCRIPT_TAG, }; +use windmill_common::client::AuthedClient; async fn process_jc( jc: JobCompleted, @@ -273,11 +273,7 @@ pub fn start_background_processor( }) } -async fn send_job_completed( - job_completed_tx: JobCompletedSender, - jc: JobCompleted, - -) { +async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) { job_completed_tx .send_job(jc, true) .with_context(windmill_common::otel_ee::otel_ctx()) @@ -301,7 +297,6 @@ pub async fn process_result( ) -> error::Result { match result { Ok(result) => { - send_job_completed( job_completed_tx, JobCompleted { diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 9f0b783471..1a6a93ecca 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,9 +19,10 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV, }; +use windmill_common::client::AuthedClient; #[cfg(windows)] use crate::SYSTEM_ROOT; diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 2adf41ccba..8828aeb76b 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -26,7 +26,8 @@ use crate::common::{ }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; -use crate::{common::build_args_values, AuthedClient}; +use crate::common::build_args_values; +use windmill_common::client::AuthedClient; #[derive(Serialize)] struct Claims { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 92fc0f0351..221650389a 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -11,6 +11,7 @@ use anyhow::anyhow; use futures::TryFutureExt; +use windmill_common::client::AuthedClient; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, @@ -28,7 +29,7 @@ use windmill_common::{ #[cfg(feature = "enterprise")] use windmill_common::ee::LICENSE_KEY_VALID; -use anyhow::{Context, Result}; +use anyhow::Result; use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; @@ -39,8 +40,7 @@ use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] use windmill_common::METRICS_ENABLED; -use reqwest::{Body, Response}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use sqlx::types::Json; use std::{ collections::HashMap, @@ -393,201 +393,6 @@ pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB pub const INIT_SCRIPT_TAG: &str = "init_script"; -#[derive(Clone)] -pub struct AuthedClient { - pub base_internal_url: String, - pub workspace: String, - pub token: String, - pub force_client: Option, -} - -impl AuthedClient { - pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result { - self.force_client - .as_ref() - .unwrap_or(&HTTP_CLIENT) - .get(url) - .query(&query) - .header( - reqwest::header::ACCEPT, - reqwest::header::HeaderValue::from_static("application/json"), - ) - .header( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?, - ) - .send() - .await - .map_err(|e| { - tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}"); - anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}") - }) - } - - pub async fn get_id_token(&self, audience: &str) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/oidc/token/{}", - self.base_internal_url, self.workspace, audience - ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding oidc token as json string")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn get_resource_value(&self, path: &str) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/resources/get_value/{}", - self.base_internal_url, self.workspace, path - ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding resource value as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn get_variable_value(&self, path: &str) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/variables/get_value/{}", - self.base_internal_url, self.workspace, path - ); - let response = self.get(&url, vec![]).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding variable value as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn get_resource_value_interpolated( - &self, - path: &str, - job_id: Option, - ) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/resources/get_value_interpolated/{}", - self.base_internal_url, self.workspace, path - ); - let mut query = Vec::with_capacity(1usize); - if let Some(v) = &job_id { - query.push(("job_id", v.to_string())); - } - let response = self.get(&url, query).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding interpolated resource value as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn get_completed_job_result( - &self, - path: &str, - json_path: Option, - ) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/jobs_u/completed/get_result/{}", - self.base_internal_url, self.workspace, path - ); - let query = if let Some(json_path) = json_path { - vec![("json_path", json_path)] - } else { - vec![] - }; - let response = self.get(&url, query).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding completed job result as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn get_result_by_id( - &self, - flow_job_id: &str, - node_id: &str, - json_path: Option, - ) -> anyhow::Result { - let url = format!( - "{}/api/w/{}/jobs/result_by_id/{}/{}", - self.base_internal_url, self.workspace, flow_job_id, node_id - ); - let query = if let Some(json_path) = json_path { - vec![("json_path", json_path)] - } else { - vec![] - }; - let response = self.get(&url, query).await?; - match response.status().as_u16() { - 200u16 => Ok(response - .json::() - .await - .context("decoding result by id as json")?), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), - } - } - - pub async fn upload_s3_file( - &self, - workspace_id: &str, - object_key: String, - storage: Option, - body: S, - ) -> error::Result<()> - where - S: futures::stream::TryStream + Send + 'static, - S::Error: Into>, - bytes::Bytes: From, - { - let mut query = vec![("file_key", object_key)]; - if let Some(storage) = storage { - query.push(("storage", storage)); - } - let response = self - .force_client - .as_ref() - .unwrap_or(&HTTP_CLIENT) - .post(format!( - "{}/api/w/{}/job_helpers/upload_s3_file", - self.base_internal_url, workspace_id - )) - .query(&query) - .header( - reqwest::header::ACCEPT, - reqwest::header::HeaderValue::from_static("application/json"), - ) - .header( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) - .map_err(|e| error::Error::BadConfig(e.to_string()))?, - ) - .body(Body::wrap_stream(body)) - .send() - .await - .context(format!("Sent upload_s3_file request",)) - .map_err(error::Error::from)?; - - match response.status().as_u16() { - 200u16 => Ok(()), - _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, - } - } -} - #[derive(Clone)] pub struct SameWorkerSender(pub Sender, pub Arc); diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 0af2bfcc72..6c0b133828 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -15,8 +15,7 @@ use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::worker_utils::get_tag_and_concurrency; use crate::{ - AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow, - KEEP_JOB_DIR, + JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow, KEEP_JOB_DIR, }; use anyhow::Context; use futures::TryFutureExt; @@ -32,6 +31,7 @@ use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; +use windmill_common::client::AuthedClient; use windmill_common::db::Authed; use windmill_common::flow_status::{ ApprovalConditions, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index d32413fc1e..6710f718c4 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1964,13 +1964,12 @@ async fn ansible_dep( ) -> std::result::Result { use windmill_parser_yaml::add_versions_to_requirements_yaml; - use crate::{ - ansible_executor::{ + use crate::ansible_executor::{ create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, install_galaxy_collections, - }, - AuthedClient, - }; + }; + use windmill_common::client::AuthedClient; + let python_lockfile = python_dep( reqs.python_reqs.join("\n").to_string(), diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index aa223e3b50..3ac0674451 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -28,7 +28,14 @@ endpoint?: string } - export let bucket_config: S3Config | AzureConfig | undefined = undefined + type AwsOidcConfig = { + type: 'AwsOidc' + bucket: string + region: string + roleArn: string + } + + export let bucket_config: S3Config | AzureConfig | AwsOidcConfig | undefined = undefined $: bucket_config?.type == 'S3' && bucket_config.allow_http == undefined && @@ -125,6 +132,7 @@ > S3 Azure Blob + AWS OIDC
{#if bucket_config.type === 'S3'} @@ -210,6 +218,23 @@ > + {:else if bucket_config.type === 'AwsOidc'} + + + {:else}
Unknown bucket type {bucket_config['type']}
{/if}