Files
windmill/backend/windmill-common/src/client.rs
Diego Imbert cd02586ba2 feat: use S3 Proxy for duckdb instead of direct connection (#6505)
* s3 proxy works with get (no auth yet)

* nit

* support s3:// syntax

* Support s3:// syntax and fix vite api proxy normalizing double slashes in URI

* s3 checks authed

* nit

* PUT works

* delete file works

* Derive the JWT signature from the backend

* Authorize s3 correctly (JWT signature is never sent in cleartext)

* convert object store error to wmill error for correct status code

* stash

* fix

* POST first request proxy works

* s3 put for duckdb

* factor out direct proxy code

* Fix Issue with backend proxy and wrong signature due to Host header mismatch

* Add _default_ syntax to solve URI normalization issues with signing

* restricted to user paths toggle

* user path restriction works !

* change restriction to allow

* fix

* factor out code

* better permissions UX in object storage settings

* Revert to restrict_to_user_paths

* check permissions in old s3 api

* DuckDB now uses S3 Proxy and no longer needs LFS query

* implement todo

* fix hardcoded w_id

* s3 proxy size limit

* s3_proxy is ee

* nit

* add Google Cloud Storage as option to secondary storage

* GCS secret in duckdb

* fix toolchain compile

* Remove user permissions for v0

* fix ci 2

* fix CI OSS

* fix missing feature flag

* fix unused warning

* integration test fails bc rustc 1.85.0

* ee ref

* fix ci ...

* update ee ref
2025-09-03 15:03:44 +00:00

210 lines
7.1 KiB
Rust

use anyhow::Context;
use reqwest::{Body, Response};
use serde::de::DeserializeOwned;
use crate::utils::HTTP_CLIENT;
#[derive(Clone)]
pub struct AuthedClient {
pub base_internal_url: String,
pub workspace: String,
pub token: String,
pub force_client: Option<reqwest::Client>,
}
impl AuthedClient {
pub fn new(
base_internal_url: String,
workspace: String,
token: String,
force_client: Option<reqwest::Client>,
) -> AuthedClient {
AuthedClient { base_internal_url, workspace, token, force_client }
}
pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
self.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.get(url)
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
)
.send()
.await
.map_err(|e| {
tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
})
}
pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/oidc/token/{}",
self.base_internal_url, self.workspace, audience
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding oidc token as json string")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/variables/get_value/{}",
self.base_internal_url, self.workspace, path
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<String>()
.await
.context("decoding variable value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
&self,
path: &str,
job_id: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value_interpolated/{}",
self.base_internal_url, self.workspace, path
);
let mut query = Vec::with_capacity(1usize);
if let Some(v) = &job_id {
query.push(("job_id", v.to_string()));
}
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding interpolated resource value as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_completed_job_result<T: DeserializeOwned>(
&self,
path: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs_u/completed/get_result/{}",
self.base_internal_url, self.workspace, path
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding completed job result as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_result_by_id<T: DeserializeOwned>(
&self,
flow_job_id: &str,
node_id: &str,
json_path: Option<String>,
) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/jobs/result_by_id/{}/{}",
self.base_internal_url, self.workspace, flow_job_id, node_id
);
let query = if let Some(json_path) = json_path {
vec![("json_path", json_path)]
} else {
vec![]
};
let response = self.get(&url, query).await?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<T>()
.await
.context("decoding result by id as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn upload_s3_file<S>(
&self,
workspace_id: &str,
object_key: String,
storage: Option<String>,
body: S,
) -> anyhow::Result<()>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
bytes::Bytes: From<S::Ok>,
{
let mut query = vec![("file_key", object_key)];
if let Some(storage) = storage {
query.push(("storage", storage));
}
let response = self
.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.post(format!(
"{}/api/w/{}/job_helpers/upload_s3_file",
self.base_internal_url, workspace_id
))
.query(&query)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
.map_err(|e| anyhow::anyhow!(e.to_string()))?,
)
.body(Body::wrap_stream(body))
.send()
.await
.context(format!("Sent upload_s3_file request",))
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
match response.status().as_u16() {
200u16 => Ok(()),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
}
}