* s3 proxy works with get (no auth yet) * nit * support s3:// syntax * Support s3:// syntax and fix vite api proxy normalizing double slashes in URI * s3 checks authed * nit * PUT works * delete file works * Derive the JWT signature from the backend * Authorize s3 correctly (JWT signature is never sent in cleartext) * convert object store error to wmill error for correct status code * stash * fix * POST first request proxy works * s3 put for duckdb * factor out direct proxy code * Fix Issue with backend proxy and wrong signature due to Host header mismatch * Add _default_ syntax to solve URI normalization issues with signing * restricted to user paths toggle * user path restriction works ! * change restriction to allow * fix * factor out code * better permissions UX in object storage settings * Revert to restrict_to_user_paths * check permissions in old s3 api * DuckDB now uses S3 Proxy and no longer needs LFS query * implement todo * fix hardcoded w_id * s3 proxy size limit * s3_proxy is ee * nit * add Google Cloud Storage as option to secondary storage * GCS secret in duckdb * fix toolchain compile * Remove user permissions for v0 * fix ci 2 * fix CI OSS * fix missing feature flag * fix unused warning * integration test fails bc rustc 1.85.0 * ee ref * fix ci ... * update ee ref
210 lines
7.1 KiB
Rust
210 lines
7.1 KiB
Rust
use anyhow::Context;
|
|
use reqwest::{Body, Response};
|
|
use serde::de::DeserializeOwned;
|
|
|
|
use crate::utils::HTTP_CLIENT;
|
|
|
|
#[derive(Clone)]
|
|
pub struct AuthedClient {
|
|
pub base_internal_url: String,
|
|
pub workspace: String,
|
|
pub token: String,
|
|
pub force_client: Option<reqwest::Client>,
|
|
}
|
|
|
|
impl AuthedClient {
|
|
pub fn new(
|
|
base_internal_url: String,
|
|
workspace: String,
|
|
token: String,
|
|
force_client: Option<reqwest::Client>,
|
|
) -> AuthedClient {
|
|
AuthedClient { base_internal_url, workspace, token, force_client }
|
|
}
|
|
|
|
pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result<Response> {
|
|
self.force_client
|
|
.as_ref()
|
|
.unwrap_or(&HTTP_CLIENT)
|
|
.get(url)
|
|
.query(&query)
|
|
.header(
|
|
reqwest::header::ACCEPT,
|
|
reqwest::header::HeaderValue::from_static("application/json"),
|
|
)
|
|
.header(
|
|
reqwest::header::AUTHORIZATION,
|
|
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
|
|
)
|
|
.send()
|
|
.await
|
|
.map_err(|e| {
|
|
tracing::error!("Error executing get request from authed http client to {url} with query {query:?}: {e}");
|
|
anyhow::anyhow!("Error executing get request from authed http client to {url} with query {query:?}: {e}")
|
|
})
|
|
}
|
|
|
|
pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
|
|
let url = format!(
|
|
"{}/api/w/{}/oidc/token/{}",
|
|
self.base_internal_url, self.workspace, audience
|
|
);
|
|
let response = self.get(&url, vec![]).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<String>()
|
|
.await
|
|
.context("decoding oidc token as json string")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
|
|
let url = format!(
|
|
"{}/api/w/{}/resources/get_value/{}",
|
|
self.base_internal_url, self.workspace, path
|
|
);
|
|
let response = self.get(&url, vec![]).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<T>()
|
|
.await
|
|
.context("decoding resource value as json")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn get_variable_value(&self, path: &str) -> anyhow::Result<String> {
|
|
let url = format!(
|
|
"{}/api/w/{}/variables/get_value/{}",
|
|
self.base_internal_url, self.workspace, path
|
|
);
|
|
let response = self.get(&url, vec![]).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<String>()
|
|
.await
|
|
.context("decoding variable value as json")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn get_resource_value_interpolated<T: DeserializeOwned>(
|
|
&self,
|
|
path: &str,
|
|
job_id: Option<String>,
|
|
) -> anyhow::Result<T> {
|
|
let url = format!(
|
|
"{}/api/w/{}/resources/get_value_interpolated/{}",
|
|
self.base_internal_url, self.workspace, path
|
|
);
|
|
let mut query = Vec::with_capacity(1usize);
|
|
if let Some(v) = &job_id {
|
|
query.push(("job_id", v.to_string()));
|
|
}
|
|
let response = self.get(&url, query).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<T>()
|
|
.await
|
|
.context("decoding interpolated resource value as json")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn get_completed_job_result<T: DeserializeOwned>(
|
|
&self,
|
|
path: &str,
|
|
json_path: Option<String>,
|
|
) -> anyhow::Result<T> {
|
|
let url = format!(
|
|
"{}/api/w/{}/jobs_u/completed/get_result/{}",
|
|
self.base_internal_url, self.workspace, path
|
|
);
|
|
let query = if let Some(json_path) = json_path {
|
|
vec![("json_path", json_path)]
|
|
} else {
|
|
vec![]
|
|
};
|
|
let response = self.get(&url, query).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<T>()
|
|
.await
|
|
.context("decoding completed job result as json")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn get_result_by_id<T: DeserializeOwned>(
|
|
&self,
|
|
flow_job_id: &str,
|
|
node_id: &str,
|
|
json_path: Option<String>,
|
|
) -> anyhow::Result<T> {
|
|
let url = format!(
|
|
"{}/api/w/{}/jobs/result_by_id/{}/{}",
|
|
self.base_internal_url, self.workspace, flow_job_id, node_id
|
|
);
|
|
let query = if let Some(json_path) = json_path {
|
|
vec![("json_path", json_path)]
|
|
} else {
|
|
vec![]
|
|
};
|
|
let response = self.get(&url, query).await?;
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(response
|
|
.json::<T>()
|
|
.await
|
|
.context("decoding result by id as json")?),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
|
|
}
|
|
}
|
|
|
|
pub async fn upload_s3_file<S>(
|
|
&self,
|
|
workspace_id: &str,
|
|
object_key: String,
|
|
storage: Option<String>,
|
|
body: S,
|
|
) -> anyhow::Result<()>
|
|
where
|
|
S: futures::stream::TryStream + Send + 'static,
|
|
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
bytes::Bytes: From<S::Ok>,
|
|
{
|
|
let mut query = vec![("file_key", object_key)];
|
|
if let Some(storage) = storage {
|
|
query.push(("storage", storage));
|
|
}
|
|
let response = self
|
|
.force_client
|
|
.as_ref()
|
|
.unwrap_or(&HTTP_CLIENT)
|
|
.post(format!(
|
|
"{}/api/w/{}/job_helpers/upload_s3_file",
|
|
self.base_internal_url, workspace_id
|
|
))
|
|
.query(&query)
|
|
.header(
|
|
reqwest::header::ACCEPT,
|
|
reqwest::header::HeaderValue::from_static("application/json"),
|
|
)
|
|
.header(
|
|
reqwest::header::AUTHORIZATION,
|
|
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
|
|
.map_err(|e| anyhow::anyhow!(e.to_string()))?,
|
|
)
|
|
.body(Body::wrap_stream(body))
|
|
.send()
|
|
.await
|
|
.context(format!("Sent upload_s3_file request",))
|
|
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
|
|
|
|
match response.status().as_u16() {
|
|
200u16 => Ok(()),
|
|
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
|
|
}
|
|
}
|
|
}
|