use std::collections::HashMap; use axum::{ extract::{FromRequest, Multipart, Query, Request}, http::HeaderMap, response::{IntoResponse, Response}, }; use bytes::Bytes; use http::{header::CONTENT_TYPE, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::types::JsonRawValue; use windmill_common::{ error::Error, triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}, worker::to_raw_value, DB, }; use windmill_queue::PushArgsOwned; use crate::{ db::ApiAuthed, triggers::trigger_helpers::{get_runnable_format, RunnableId}, }; #[derive(Debug)] pub enum RawBody { Json(String), CEJson(String), Text(String), Xml(String), UrlEncoded(Bytes), Multipart(Multipart), Empty, } #[derive(Debug, Clone, Serialize)] #[serde(untagged)] pub enum Body { HashMap(HashMap>), NoHashMap(Box), } #[derive(Debug, Clone, Default)] pub struct WebhookArgsMetadata { pub raw_string: Option, pub headers: HeaderMap, pub query: Option, pub method: http::Method, pub query_wrap_body: bool, pub query_use_raw: bool, pub query_include_header: Option, pub query_include_query: Option, } pub struct RawWebhookArgs { pub body: RawBody, pub metadata: WebhookArgsMetadata, } #[derive(Debug, Clone)] pub struct WebhookArgs { pub body: Body, pub metadata: WebhookArgsMetadata, } // capture // impl RawWebhookArgs { #[cfg(not(feature = "parquet"))] pub async fn process_multipart( _multipart: Multipart, _authed: &ApiAuthed, _db: &DB, _w_id: &str, ) -> Result>, Error> { return Err(Error::BadRequest(format!( "multipart/form-data requires the parquet feature" ))); } #[cfg(feature = "parquet")] async fn process_multipart( mut multipart: Multipart, authed: &ApiAuthed, db: &DB, w_id: &str, ) -> Result>, Error> { use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; use futures::TryStreamExt; use object_store::{Attribute, Attributes}; use windmill_common::s3_helpers::build_object_store_client; let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, w_id, None).await?; if let Some(s3_resource) = s3_resource { let s3_client = build_object_store_client(&s3_resource).await?; let mut body = HashMap::new(); let mut files = HashMap::new(); while let Some(field) = multipart.next_field().await.map_err(|e| { Error::BadRequest(format!("Error reading multipart field: {}", e.body_text())) })? { if let Some(name) = field.name().map(|x| x.to_string()) { if let Some(content_type) = field.content_type() { let ext = field .file_name() .map(|x| x.split('.').last()) .flatten() .map(|x| x.to_string()); let file_key = get_random_file_name(ext); let options = Attributes::from_iter(vec![ (Attribute::ContentType, content_type.to_string()), ( Attribute::ContentDisposition, if let Some(filename) = field.file_name() { format!("inline; filename=\"{}\"", filename) } else { "inline".to_string() }, ), ]) .into(); let bytes_stream = field .into_stream() .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) .await?; files.entry(name).or_insert(vec![]).push(serde_json::json!({ "s3": &file_key })); } else { body.insert(name, to_raw_value(&field.text().await.unwrap_or_default())); } } } for (k, v) in files { body.insert(k, to_raw_value(&v)); } Ok(body) } else { Err(Error::BadRequest(format!( "You need to connect your workspace to an S3 bucket to use multipart/form-data" ))) } } pub async fn process_args( self, authed: &ApiAuthed, db: &DB, w_id: &str, force_use_raw: Option, ) -> Result { let use_raw = force_use_raw.unwrap_or(self.metadata.query_use_raw); match self.body { RawBody::Multipart(multipart) => { let body = Self::process_multipart(multipart, authed, db, w_id).await?; Ok(WebhookArgs { body: Body::HashMap(body), metadata: self.metadata }) } RawBody::Empty => { let mut metadata = self.metadata; if use_raw { metadata.raw_string = Some("".to_string()); } Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata }) } RawBody::Text(s) | RawBody::Xml(s) => Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata: WebhookArgsMetadata { raw_string: Some(s), ..self.metadata }, }), RawBody::UrlEncoded(bytes) => { let mut metadata = self.metadata; if use_raw { let raw_string = String::from_utf8(bytes.to_vec()) .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)))?; metadata.raw_string = Some(raw_string); } let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) .map_err(|e| Error::BadRequest(format!("invalid urlencoded data: {}", e)))?; let payload = payload .into_iter() .map(|(k, v)| (k, to_raw_value(&v))) .collect::>(); Ok(WebhookArgs { body: Body::HashMap(payload), metadata }) } RawBody::Json(s) => WebhookArgs::from_json(self.metadata, use_raw, s).await, RawBody::CEJson(s) => WebhookArgs::from_ce_json(self.metadata, use_raw, s).await, } } pub async fn to_main_args( self, authed: &ApiAuthed, db: &DB, w_id: &str, ) -> Result { let args = self.process_args(authed, db, w_id, None).await?; args.to_main_args() } pub async fn to_args_from_runnable( self, authed: &ApiAuthed, db: &DB, w_id: &str, runnable_id: RunnableId, skip_preprocessor: Option, ) -> Result { let args = self.process_args(authed, db, w_id, None).await?; args.to_args_from_runnable(db, w_id, runnable_id, skip_preprocessor) .await } } #[derive(Serialize)] struct WebhookPreprocessorEvent { kind: String, body: Box, raw_string: Option, headers: HashMap>, query: HashMap>, } impl WebhookArgs { pub fn to_main_args(self) -> Result { self.to_args_from_format(RunnableFormat { has_preprocessor: false, version: RunnableFormatVersion::V2, }) } pub async fn to_args_from_runnable( self, db: &DB, w_id: &str, runnable_id: RunnableId, skip_preprocessor: Option, ) -> Result { if skip_preprocessor.unwrap_or(false) { self.to_main_args() } else { let runnable_format = get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; self.to_args_from_format(runnable_format) } } pub fn to_args_from_format( self, runnable_format: RunnableFormat, ) -> Result { let headers = build_headers( &self.metadata.headers, self.metadata.query_include_header, runnable_format.has_preprocessor, ); let query = build_query( self.metadata.query.as_deref(), self.metadata.query_include_query, runnable_format.has_preprocessor, ); match runnable_format { RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { let mut args = HashMap::new(); args.insert( "event".to_string(), to_raw_value(&WebhookPreprocessorEvent { kind: "webhook".to_string(), body: to_raw_value(&self.body), raw_string: self.metadata.raw_string, headers, query, }), ); Ok(PushArgsOwned { args, extra: None }) } RunnableFormat { has_preprocessor, .. } => { let mut extra = HashMap::new(); let WebhookArgsMetadata { query_wrap_body, raw_string, .. } = self.metadata; for (k, v) in headers { extra.insert(k, v); } for (k, v) in query { extra.insert(k, v); } if let Some(raw_string) = raw_string { extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); } if has_preprocessor { // if has preprocessor, it has to be v1 extra.insert( "wm_trigger".to_string(), to_raw_value(&serde_json::json!({ "kind": "webhook", })), ); } let extra = if extra.is_empty() { None } else { Some(extra) }; match self.body { Body::HashMap(mut body) => { if query_wrap_body { body = HashMap::from([("body".to_string(), to_raw_value(&body))]); } Ok(PushArgsOwned { args: body, extra }) } Body::NoHashMap(args) => { let mut hm = HashMap::new(); hm.insert("body".to_string(), args); Ok(PushArgsOwned { args: hm, extra }) } } } } } } #[derive(Deserialize)] pub struct RequestQuery { pub raw: Option, pub wrap_body: Option, pub include_header: Option, pub include_query: Option, } async fn req_to_string( req: Request, _state: &S, ) -> Result { let bytes = Bytes::from_request(req, _state) .await .map_err(IntoResponse::into_response)?; String::from_utf8(bytes.to_vec()) .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response()) } pub async fn try_from_request_body( request: Request, _state: &S, is_http_trigger: bool, ) -> Result where S: Send + Sync, { let (content_type, metadata) = { let headers_map = request.headers(); let content_type_header = headers_map.get(CONTENT_TYPE); let content_type = content_type_header.and_then(|value| value.to_str().ok()); let uri = request.uri(); let request_query = Query::::try_from_uri(uri).unwrap().0; let query = uri.query().map(|s| s.to_owned()); let raw = !is_http_trigger && request_query.raw.unwrap_or(false); let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false); ( content_type, WebhookArgsMetadata { headers: headers_map.clone(), query, method: request.method().clone(), raw_string: None, query_wrap_body: wrap_body, query_use_raw: raw, query_include_header: request_query.include_header, query_include_query: request_query.include_query, }, ) }; let no_content_type = content_type.is_none(); if no_content_type || content_type.unwrap().starts_with("application/json") { let bytes = Bytes::from_request(request, _state) .await .map_err(IntoResponse::into_response)?; if no_content_type && bytes.is_empty() { Ok(RawWebhookArgs { body: RawBody::Empty, metadata }) } else { let str = String::from_utf8(bytes.to_vec()) .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; Ok(RawWebhookArgs { body: RawBody::Json(str), metadata }) } } else if content_type .unwrap() .starts_with("application/cloudevents+json") { let str = req_to_string(request, _state).await?; Ok(RawWebhookArgs { body: RawBody::CEJson(str), metadata }) } else if content_type .unwrap() .starts_with("application/cloudevents-batch+json") { Err( Error::BadRequest(format!("Cloud events batching is not supported yet")) .into_response(), ) } else if content_type.unwrap().starts_with("text/plain") { let str = req_to_string(request, _state).await?; Ok(RawWebhookArgs { body: RawBody::Text(str), metadata }) } else if content_type .unwrap() .starts_with("application/x-www-form-urlencoded") { let bytes = Bytes::from_request(request, _state) .await .map_err(IntoResponse::into_response)?; Ok(RawWebhookArgs { body: RawBody::UrlEncoded(bytes), metadata }) } else if content_type.unwrap().starts_with("application/xml") || content_type.unwrap().starts_with("text/xml") { let str = req_to_string(request, _state).await?; Ok(RawWebhookArgs { body: RawBody::Xml(str), metadata }) } else if content_type.unwrap().starts_with("multipart/form-data") { let multipart = Multipart::from_request(request, _state) .await .map_err(IntoResponse::into_response)?; Ok(RawWebhookArgs { body: RawBody::Multipart(multipart), metadata }) } else { Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) } } #[axum::async_trait] impl FromRequest for RawWebhookArgs where S: Send + Sync, { type Rejection = Response; async fn from_request(request: Request, _state: &S) -> Result { let args = try_from_request_body(request, _state, false).await?; Ok(args) } } lazy_static::lazy_static! { static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") .ok().map(|x| x .split(',') .map(|s| s.to_string()) .collect()).unwrap_or_default(); } pub fn build_headers( headers: &HeaderMap, include_header: Option, include_all_headers: bool, ) -> HashMap> { let mut selected_headers = HashMap::new(); if include_all_headers { for (k, v) in headers.iter() { selected_headers.insert( k.to_string(), to_raw_value(&v.to_str().unwrap_or("").to_string()), ); } } else { let whitelist = include_header .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) .unwrap_or_default(); whitelist .iter() .chain(INCLUDE_HEADERS.iter()) .for_each(|h| { if let Some(v) = headers.get(h) { selected_headers.insert( h.to_string().to_lowercase().replace('-', "_"), to_raw_value(&v.to_str().unwrap_or("").to_string()), ); } }); } selected_headers } pub fn build_query( query: Option<&str>, include_query: Option, include_all_query: bool, ) -> HashMap> { let Some(query) = query else { return HashMap::new(); }; if include_all_query { let queries = serde_urlencoded::from_str::>(&query).unwrap_or_default(); queries .into_iter() .map(|(k, v)| (k, to_raw_value(&v))) .collect() } else { let parse_query_args = include_query .map(|s| s.split(",").map(|p| p.to_string()).collect::>()) .unwrap_or_default(); let mut args = HashMap::new(); if !parse_query_args.is_empty() { let queries = serde_urlencoded::from_str::>(&query).unwrap_or_default(); parse_query_args.iter().for_each(|h| { if let Some(v) = queries.get(h) { args.insert(h.to_string(), to_raw_value(v)); } }); } args } } fn restructure_cloudevents_metadata( mut p: HashMap>, ) -> Result>, Error> { let data = p .remove("data") .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); let str = data.to_string(); let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{'; if wrap_body { let args = serde_json::from_str::>>(&str) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); let mut hm = HashMap::new(); hm.insert("body".to_string(), args); hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); Ok(hm) } else { let mut hm = serde_json::from_str::>>>(&str) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(HashMap::new); hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); Ok(hm) } } impl WebhookArgs { async fn from_json( mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, ) -> Result { if use_raw { metadata.raw_string = Some(str.clone()); } let no_hashmap = str.len() > 0 && str.chars().next().unwrap() != '{'; if no_hashmap { let args = serde_json::from_str::>>(&str) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); Ok(Self { body: Body::NoHashMap(args), metadata }) } else { let hm = serde_json::from_str::>>>(&str) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(HashMap::new); Ok(Self { body: Body::HashMap(hm), metadata }) } } async fn from_ce_json( mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, ) -> Result { if use_raw { metadata.raw_string = Some(str.clone()); } let hm = serde_json::from_str::>>(&str) .map_err(|e| Error::BadRequest(format!("invalid cloudevents+json: {}", e)))?; let hm = restructure_cloudevents_metadata(hm)?; Ok(Self { body: Body::HashMap(hm), metadata }) } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_cloudevents_json_payload() { let r1 = r#" { "specversion" : "1.0", "type" : "com.example.someevent", "source" : "/mycontext", "subject": null, "id" : "C234-1234-1234", "time" : "2018-04-05T17:31:00Z", "comexampleextension1" : "value", "comexampleothervalue" : 5, "datacontenttype" : "application/json", "data" : { "appinfoA" : "abc", "appinfoB" : 123, "appinfoC" : true } } "#; let r2 = r#" { "specversion" : "1.0", "type" : "com.example.someevent", "source" : "/mycontext", "subject": null, "id" : "C234-1234-1234", "time" : "2018-04-05T17:31:00Z", "comexampleextension1" : "value", "comexampleothervalue" : 5, "datacontenttype" : "application/json", "data" : 1.5 } "#; let metadata = WebhookArgsMetadata::default(); let a1 = WebhookArgs::from_ce_json(metadata.clone(), false, r1.to_string()) .await .expect("Failed to parse the cloudevent"); let a2 = WebhookArgs::from_ce_json(metadata.clone(), false, r2.to_string()) .await .expect("Failed to parse the cloudevent"); match a1.body { Body::HashMap(body) => { body.get("WEBHOOK__METADATA__").expect( "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", ); } _ => panic!("Expected a HashMap"), } match a2.body { Body::HashMap(body) => { assert_eq!( body .get("body") .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") .to_string(), "1.5" ); } _ => panic!("Expected a HashMap"), } } }