add proto postgresql backend support as a job language
This commit is contained in:
85
backend/Cargo.lock
generated
85
backend/Cargo.lock
generated
@@ -149,6 +149,12 @@ dependencies = [
|
||||
"password-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "array-init"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
|
||||
|
||||
[[package]]
|
||||
name = "ascii-canvas"
|
||||
version = "3.0.0"
|
||||
@@ -244,7 +250,7 @@ dependencies = [
|
||||
"polling",
|
||||
"rustix",
|
||||
"slab",
|
||||
"socket2",
|
||||
"socket2 0.4.9",
|
||||
"waker-fn",
|
||||
]
|
||||
|
||||
@@ -1202,6 +1208,12 @@ version = "2.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "1.9.0"
|
||||
@@ -1729,7 +1741,7 @@ dependencies = [
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"socket2 0.4.9",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -2747,6 +2759,38 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postgres-protocol"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d"
|
||||
dependencies = [
|
||||
"base64 0.21.2",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"hmac",
|
||||
"md-5 0.10.5",
|
||||
"memchr",
|
||||
"rand 0.8.5",
|
||||
"sha2 0.10.7",
|
||||
"stringprep",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postgres-types"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6"
|
||||
dependencies = [
|
||||
"array-init",
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"postgres-protocol",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
@@ -3776,6 +3820,16 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sourcemap"
|
||||
version = "6.2.3"
|
||||
@@ -4329,7 +4383,7 @@ dependencies = [
|
||||
"parking_lot 0.12.1",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"socket2 0.4.9",
|
||||
"tokio-macros",
|
||||
"tracing",
|
||||
"windows-sys 0.48.0",
|
||||
@@ -4367,6 +4421,30 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-postgres"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"log",
|
||||
"parking_lot 0.12.1",
|
||||
"percent-encoding",
|
||||
"phf 0.11.1",
|
||||
"pin-project-lite",
|
||||
"postgres-protocol",
|
||||
"postgres-types",
|
||||
"socket2 0.5.3",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.23.4"
|
||||
@@ -5449,6 +5527,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sqlx",
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tracing",
|
||||
"uuid 1.3.4",
|
||||
"windmill-api-client",
|
||||
|
||||
@@ -175,4 +175,5 @@ wasm-bindgen = "0.2"
|
||||
serde-wasm-bindgen = "0.4"
|
||||
wasm-bindgen-test = "0.3.0"
|
||||
convert_case = "0.6.0"
|
||||
getrandom = "0.2"
|
||||
getrandom = "0.2"
|
||||
tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1"]}
|
||||
@@ -0,0 +1 @@
|
||||
-- Add down migration script here
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Add up migration script here
|
||||
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'postgresql';
|
||||
@@ -2151,7 +2151,7 @@ paths:
|
||||
schema: {}
|
||||
language:
|
||||
type: string
|
||||
enum: [deno, python3, go, bash]
|
||||
enum: [deno, python3, go, bash, postgresql]
|
||||
summary:
|
||||
type: string
|
||||
required:
|
||||
@@ -5531,7 +5531,7 @@ components:
|
||||
type: string
|
||||
language:
|
||||
type: string
|
||||
enum: [python3, deno, go, bash]
|
||||
enum: [python3, deno, go, bash, postgresql]
|
||||
kind:
|
||||
type: string
|
||||
enum: [script, failure, trigger, command, approval]
|
||||
@@ -5730,7 +5730,6 @@ components:
|
||||
"script_hub",
|
||||
"identity",
|
||||
"http",
|
||||
"postgresql",
|
||||
"graphql",
|
||||
]
|
||||
schedule_path:
|
||||
@@ -5823,7 +5822,6 @@ components:
|
||||
"identity",
|
||||
"http",
|
||||
"graphql",
|
||||
"postgresql",
|
||||
]
|
||||
schedule_path:
|
||||
type: string
|
||||
@@ -6273,7 +6271,7 @@ components:
|
||||
type: string
|
||||
kind:
|
||||
type: string
|
||||
enum: [code, identity, graphql, postgresql, http]
|
||||
enum: [code, identity, graphql, http]
|
||||
|
||||
required:
|
||||
- args
|
||||
|
||||
@@ -1240,7 +1240,6 @@ enum PreviewKind {
|
||||
Code,
|
||||
Identity,
|
||||
Http,
|
||||
Postgresql,
|
||||
Graphql,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
@@ -1977,7 +1976,6 @@ async fn run_preview_job(
|
||||
Some(PreviewKind::Identity) => JobPayload::Identity,
|
||||
Some(PreviewKind::Http) => JobPayload::Http,
|
||||
Some(PreviewKind::Graphql) => JobPayload::Graphql,
|
||||
Some(PreviewKind::Postgresql) => JobPayload::Postgresql,
|
||||
_ => JobPayload::Code(RawCode {
|
||||
content: preview.content.unwrap_or_default(),
|
||||
path: preview.path,
|
||||
|
||||
@@ -1302,6 +1302,7 @@ async fn tarball_workspace(
|
||||
ScriptLang::Deno => "ts",
|
||||
ScriptLang::Go => "go",
|
||||
ScriptLang::Bash => "sh",
|
||||
ScriptLang::Postgresql => "psql",
|
||||
};
|
||||
archive
|
||||
.write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
|
||||
|
||||
@@ -276,7 +276,6 @@ pub enum FlowModuleValue {
|
||||
language: ScriptLang,
|
||||
},
|
||||
Identity,
|
||||
Postgresql,
|
||||
Http,
|
||||
Graphql,
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ pub enum JobKind {
|
||||
Identity,
|
||||
FlowDependencies,
|
||||
Http,
|
||||
Postgresql,
|
||||
Graphql,
|
||||
}
|
||||
|
||||
@@ -160,7 +159,6 @@ pub enum JobPayload {
|
||||
RawFlow { value: FlowValue, path: Option<String> },
|
||||
Identity,
|
||||
Http,
|
||||
Postgresql,
|
||||
Graphql,
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ pub enum ScriptLang {
|
||||
Python3,
|
||||
Go,
|
||||
Bash,
|
||||
Postgresql,
|
||||
}
|
||||
|
||||
impl ScriptLang {
|
||||
@@ -38,6 +39,7 @@ impl ScriptLang {
|
||||
ScriptLang::Python3 => "python3",
|
||||
ScriptLang::Go => "go",
|
||||
ScriptLang::Bash => "bash",
|
||||
ScriptLang::Postgresql => "postgresql",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ lazy_static::lazy_static! {
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"postgresql".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
@@ -947,7 +948,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
JobPayload::Identity => (None, None, None, JobKind::Identity, None, None),
|
||||
JobPayload::Graphql => (None, None, None, JobKind::Graphql, None, None),
|
||||
JobPayload::Http => (None, None, None, JobKind::Http, None, None),
|
||||
JobPayload::Postgresql => (None, None, None, JobKind::Postgresql, None, None),
|
||||
};
|
||||
|
||||
let is_running = same_worker;
|
||||
@@ -1082,7 +1082,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
JobKind::Identity => "jobs.run.identity",
|
||||
JobKind::Http => "jobs.run.http",
|
||||
JobKind::Graphql => "jobs.run.graphql",
|
||||
JobKind::Postgresql => "jobs.run.postgresql",
|
||||
JobKind::FlowDependencies => "jobs.run.flow_dependencies",
|
||||
};
|
||||
|
||||
|
||||
@@ -51,3 +51,4 @@ git-version.workspace = true
|
||||
dyn-iter.workspace = true
|
||||
once_cell.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
tokio-postgres.workspace = true
|
||||
@@ -1,6 +1,12 @@
|
||||
use anyhow::{Context, Error};
|
||||
use serde_json::Map;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
use windmill_common::error::{self, Error};
|
||||
use tokio_postgres::{
|
||||
types::{FromSql, Type},
|
||||
Column, Row,
|
||||
};
|
||||
use windmill_common::error::{self};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
use crate::MAX_RESULT_SIZE;
|
||||
@@ -10,11 +16,11 @@ pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
|
||||
let mut content = "".to_string();
|
||||
file.read_to_string(&mut content).await?;
|
||||
if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
|
||||
return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
|
||||
}
|
||||
serde_json::from_str(&content)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -39,3 +45,139 @@ pub fn capitalize(s: &str) -> String {
|
||||
Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn postgres_row_to_json_value(row: Row) -> Result<JSONValue, Error> {
|
||||
let row_data = postgres_row_to_row_data(row)?;
|
||||
Ok(JSONValue::Object(row_data))
|
||||
}
|
||||
|
||||
// some type-aliases I use in my project
|
||||
pub type JSONValue = serde_json::Value;
|
||||
pub type RowData = Map<String, JSONValue>;
|
||||
|
||||
pub fn postgres_row_to_row_data(row: Row) -> Result<RowData, Error> {
|
||||
let mut result: Map<String, JSONValue> = Map::new();
|
||||
for (i, column) in row.columns().iter().enumerate() {
|
||||
let name = column.name();
|
||||
let json_value = pg_cell_to_json_value(&row, column, i)?;
|
||||
result.insert(name.to_string(), json_value);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn pg_cell_to_json_value(
|
||||
row: &Row,
|
||||
column: &Column,
|
||||
column_i: usize,
|
||||
) -> Result<JSONValue, Error> {
|
||||
let f64_to_json_number = |raw_val: f64| -> Result<JSONValue, Error> {
|
||||
let temp = serde_json::Number::from_f64(raw_val.into())
|
||||
.ok_or(anyhow::anyhow!("invalid json-float"))?;
|
||||
Ok(JSONValue::Number(temp))
|
||||
};
|
||||
Ok(match *column.type_() {
|
||||
// for rust-postgres <> postgres type-mappings: https://docs.rs/postgres/latest/postgres/types/trait.FromSql.html#types
|
||||
// for postgres types: https://www.postgresql.org/docs/7.4/datatype.html#DATATYPE-TABLE
|
||||
|
||||
// single types
|
||||
Type::BOOL => get_basic(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
|
||||
Type::INT2 => get_basic(row, column, column_i, |a: i16| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::INT4 => get_basic(row, column, column_i, |a: i32| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::INT8 => get_basic(row, column, column_i, |a: i64| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::TEXT | Type::VARCHAR => {
|
||||
get_basic(row, column, column_i, |a: String| Ok(JSONValue::String(a)))?
|
||||
}
|
||||
Type::JSON | Type::JSONB => get_basic(row, column, column_i, |a: JSONValue| Ok(a))?,
|
||||
Type::FLOAT4 => get_basic(row, column, column_i, |a: f32| {
|
||||
Ok(f64_to_json_number(a.into())?)
|
||||
})?,
|
||||
Type::FLOAT8 => get_basic(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))?,
|
||||
// these types require a custom StringCollector struct as an intermediary (see struct at bottom)
|
||||
Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| {
|
||||
Ok(JSONValue::String(a.0))
|
||||
})?,
|
||||
|
||||
// array types
|
||||
Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?,
|
||||
Type::INT2_ARRAY => get_array(row, column, column_i, |a: i16| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::INT4_ARRAY => get_array(row, column, column_i, |a: i32| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::INT8_ARRAY => get_array(row, column, column_i, |a: i64| {
|
||||
Ok(JSONValue::Number(serde_json::Number::from(a)))
|
||||
})?,
|
||||
Type::TEXT_ARRAY | Type::VARCHAR_ARRAY => {
|
||||
get_array(row, column, column_i, |a: String| Ok(JSONValue::String(a)))?
|
||||
}
|
||||
Type::JSON_ARRAY | Type::JSONB_ARRAY => {
|
||||
get_array(row, column, column_i, |a: JSONValue| Ok(a))?
|
||||
}
|
||||
Type::FLOAT4_ARRAY => get_array(row, column, column_i, |a: f32| {
|
||||
Ok(f64_to_json_number(a.into())?)
|
||||
})?,
|
||||
Type::FLOAT8_ARRAY => {
|
||||
get_array(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))?
|
||||
}
|
||||
// these types require a custom StringCollector struct as an intermediary (see struct at bottom)
|
||||
Type::TS_VECTOR_ARRAY => get_array(row, column, column_i, |a: StringCollector| {
|
||||
Ok(JSONValue::String(a.0))
|
||||
})?,
|
||||
|
||||
_ => get_basic(row, column, column_i, |a: String| Ok(JSONValue::String(a)))?,
|
||||
})
|
||||
}
|
||||
|
||||
fn get_basic<'a, T: FromSql<'a>>(
|
||||
row: &'a Row,
|
||||
column: &Column,
|
||||
column_i: usize,
|
||||
val_to_json_val: impl Fn(T) -> Result<JSONValue, Error>,
|
||||
) -> Result<JSONValue, Error> {
|
||||
let raw_val = row
|
||||
.try_get::<_, Option<T>>(column_i)
|
||||
.with_context(|| format!("column_name:{}", column.name()))?;
|
||||
raw_val.map_or(Ok(JSONValue::Null), val_to_json_val)
|
||||
}
|
||||
fn get_array<'a, T: FromSql<'a>>(
|
||||
row: &'a Row,
|
||||
column: &Column,
|
||||
column_i: usize,
|
||||
val_to_json_val: impl Fn(T) -> Result<JSONValue, Error>,
|
||||
) -> Result<JSONValue, Error> {
|
||||
let raw_val_array = row
|
||||
.try_get::<_, Option<Vec<T>>>(column_i)
|
||||
.with_context(|| format!("column_name:{}", column.name()))?;
|
||||
Ok(match raw_val_array {
|
||||
Some(val_array) => {
|
||||
let mut result = vec![];
|
||||
for val in val_array {
|
||||
result.push(val_to_json_val(val)?);
|
||||
}
|
||||
JSONValue::Array(result)
|
||||
}
|
||||
None => JSONValue::Null,
|
||||
})
|
||||
}
|
||||
|
||||
// you can remove this section if not using TS_VECTOR (or other types requiring an intermediary `FromSQL` struct)
|
||||
struct StringCollector(String);
|
||||
impl FromSql<'_> for StringCollector {
|
||||
fn from_sql(
|
||||
_: &Type,
|
||||
raw: &[u8],
|
||||
) -> Result<StringCollector, Box<dyn std::error::Error + Sync + Send>> {
|
||||
let result = std::str::from_utf8(raw)?;
|
||||
Ok(StringCollector(result.to_owned()))
|
||||
}
|
||||
fn accepts(_ty: &Type) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ use itertools::Itertools;
|
||||
use once_cell::sync::OnceCell;
|
||||
use serde::Deserialize;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio_postgres::NoTls;
|
||||
use windmill_api_client::{Client, types::CompletedJob};
|
||||
use windmill_parser::Typ;
|
||||
use std::{
|
||||
@@ -64,7 +65,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error};
|
||||
use crate::{
|
||||
worker_flow::{
|
||||
handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress,
|
||||
}, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs}, go_executor::{handle_go_job, install_go_dependencies},
|
||||
}, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs, postgres_row_to_json_value}, go_executor::{handle_go_job, install_go_dependencies},
|
||||
};
|
||||
|
||||
|
||||
@@ -338,7 +339,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
Some(ScriptLang::Python3),
|
||||
Some(ScriptLang::Deno),
|
||||
Some(ScriptLang::Go),
|
||||
Some(ScriptLang::Bash)];
|
||||
Some(ScriptLang::Bash),
|
||||
Some(ScriptLang::Postgresql)];
|
||||
|
||||
let worker_execution_duration: HashMap<_, _> = all_langs.clone().into_iter().map(|x| (x.clone(), prometheus::register_histogram!(
|
||||
prometheus::HistogramOpts::new(
|
||||
@@ -453,8 +455,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
let rsmq2 = rsmq.clone();
|
||||
let worker_name2 = worker_name.clone();
|
||||
let send_result = tokio::spawn(async move {
|
||||
while let Some(JobCompleted { job, logs, result}) = job_completed_rx.recv().await {
|
||||
if let Err(e) = add_completed_job(&db2, &job, true, false, result, logs, rsmq2.clone()).await {
|
||||
while let Some(JobCompleted { job, logs, result, success}) = job_completed_rx.recv().await {
|
||||
if let Err(e) = add_completed_job(&db2, &job, success, false, result, logs, rsmq2.clone()).await {
|
||||
tracing::error!(worker = %worker_name2, "failed to add completed job: {}", e);
|
||||
}
|
||||
}
|
||||
@@ -878,6 +880,7 @@ struct JobCompleted {
|
||||
job: QueuedJob,
|
||||
result: serde_json::Value,
|
||||
logs: String,
|
||||
success: bool
|
||||
}
|
||||
|
||||
fn hash_args(v: &serde_json::Value) -> i64 {
|
||||
@@ -891,6 +894,8 @@ struct HttpArgs {
|
||||
url: String
|
||||
}
|
||||
|
||||
|
||||
|
||||
async fn do_http_req(job: QueuedJob) -> windmill_common::error::Result<JobCompleted> {
|
||||
let http_args: HttpArgs = serde_json::from_value(job.args.clone().unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
@@ -904,8 +909,49 @@ async fn do_http_req(job: QueuedJob) -> windmill_common::error::Result<JobComple
|
||||
job: job,
|
||||
result: res,
|
||||
logs: "".to_string(),
|
||||
success: true
|
||||
});
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PostgresqlArgs {
|
||||
database_url: String,
|
||||
query: String,
|
||||
args: Option<Vec<String>>,
|
||||
}
|
||||
async fn do_postgresql(job: QueuedJob) -> windmill_common::error::Result<JobCompleted> {
|
||||
let pg_args: PostgresqlArgs = serde_json::from_value(job.args.clone().unwrap_or_else(|| json!({})))
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
let (client, connection) =
|
||||
tokio_postgres::connect(&pg_args.database_url, NoTls).await.map_err(to_anyhow)?;
|
||||
|
||||
|
||||
// The connection object performs the actual communication with the database,
|
||||
// so spawn it off to run on its own.
|
||||
let handle = tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
eprintln!("connection error: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
use tokio_postgres::types::ToSql;
|
||||
// Now we can execute a simple statement that just returns its parameter.
|
||||
let rows = client
|
||||
.query(&pg_args.query, &pg_args.args
|
||||
.iter()
|
||||
.map(|x| x as &(dyn ToSql + Sync))
|
||||
.collect::<Vec<&(dyn ToSql + Sync)>>())
|
||||
.await.map_err(to_anyhow)?;
|
||||
handle.abort();
|
||||
// And then check that we got back the same string we sent over.
|
||||
return Ok(JobCompleted {
|
||||
job: job,
|
||||
result: json!(rows.into_iter().map(postgres_row_to_json_value).collect::<Result<Vec<_>, _>>()?),
|
||||
logs: "".to_string(),
|
||||
success: true
|
||||
});
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
job: QueuedJob,
|
||||
@@ -945,11 +991,37 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
_ => {
|
||||
if matches!(job.job_kind, JobKind::Http) {
|
||||
tokio::task::spawn(async move {
|
||||
let jc = do_http_req(job).await.expect("do http req");
|
||||
job_completed_tx.send(jc).await.expect("send job completed");
|
||||
});
|
||||
return Ok(());
|
||||
let jc = do_http_req(job.clone()).await;
|
||||
match jc {
|
||||
Ok(jc) => job_completed_tx.send(jc).await.expect("send job completed"),
|
||||
Err(e) => job_completed_tx.send(JobCompleted {
|
||||
job: job,
|
||||
result: extract_error_value(&e.to_string(), 1),
|
||||
logs: "".to_string(),
|
||||
success: false
|
||||
}).await.expect("send job completed"),
|
||||
};
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if job.language == Some(ScriptLang::Postgresql) {
|
||||
tokio::task::spawn(async move {
|
||||
let jc = do_postgresql(job.clone()).await;
|
||||
match jc {
|
||||
Ok(jc) => job_completed_tx.send(jc).await.expect("send job completed"),
|
||||
Err(e) => job_completed_tx.send(JobCompleted {
|
||||
job: job,
|
||||
result: extract_error_value(&e.to_string(), 1),
|
||||
logs: "".to_string(),
|
||||
success: false
|
||||
}).await.expect("send job completed"),
|
||||
};
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
let mut logs = "".to_string();
|
||||
// println!("handle queue {:?}", SystemTime::now());
|
||||
if let Some(log_str) = &job.logs {
|
||||
@@ -1041,7 +1113,6 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
panic!("should not be here")
|
||||
},
|
||||
JobKind::Graphql => todo!(),
|
||||
JobKind::Postgresql => todo!(),
|
||||
_ => {
|
||||
handle_code_execution_job(
|
||||
&job,
|
||||
@@ -1105,7 +1176,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
}
|
||||
} else {
|
||||
// in the happy path and if job not a flow step, we can delegate updating the completed job in the background
|
||||
job_completed_tx.send(JobCompleted{job,result:r,logs:logs,}).await.expect("send job completed");
|
||||
job_completed_tx.send(JobCompleted{job,result:r,logs:logs, success: true}).await.expect("send job completed");
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1340,7 +1411,7 @@ mount {{
|
||||
return Err(Error::ExecutionErr(
|
||||
"Require language to be not null".to_string(),
|
||||
))?;
|
||||
}
|
||||
},
|
||||
Some(ScriptLang::Python3) => {
|
||||
handle_python_job(
|
||||
requirements_o,
|
||||
@@ -1402,7 +1473,8 @@ mount {{
|
||||
envs
|
||||
)
|
||||
.await
|
||||
}
|
||||
},
|
||||
_ => panic!("unreachable"),
|
||||
};
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
@@ -1924,7 +1996,8 @@ async fn capture_dependency_job(
|
||||
ScriptLang::Deno => {
|
||||
Ok(String::new())
|
||||
// generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, timeout).await
|
||||
}
|
||||
},
|
||||
ScriptLang::Postgresql => Ok("".to_owned()),
|
||||
ScriptLang::Bash => Ok("".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1695,7 +1695,6 @@ async fn compute_next_flow_transform(
|
||||
FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity),
|
||||
FlowModuleValue::Graphql => trivial_next_job(JobPayload::Graphql),
|
||||
FlowModuleValue::Http => trivial_next_job(JobPayload::Http),
|
||||
FlowModuleValue::Postgresql => trivial_next_job(JobPayload::Postgresql),
|
||||
FlowModuleValue::Flow { path, .. } => {
|
||||
let payload = JobPayload::Flow(path.to_string());
|
||||
Ok(NextFlowTransform::Continue(
|
||||
|
||||
@@ -190,6 +190,15 @@ while (cont) {
|
||||
identity: "itsme",
|
||||
},
|
||||
};
|
||||
} else if (config.scriptPattern == "postgresql") {
|
||||
payload = {
|
||||
path: "postgresql",
|
||||
language: "postgresql",
|
||||
args: {
|
||||
query: "SELECT email FROM usr",
|
||||
database_url: "postgres://postgres:changeme@localhost:5432/windmill",
|
||||
},
|
||||
};
|
||||
} else {
|
||||
payload = {
|
||||
path: "denosimple",
|
||||
|
||||
@@ -114,7 +114,6 @@
|
||||
} else {
|
||||
await runnableComponent?.runComponent()
|
||||
}
|
||||
console.log('close')
|
||||
modal?.close()
|
||||
}}
|
||||
size="xs"
|
||||
|
||||
@@ -160,7 +160,6 @@ components:
|
||||
- $ref: "#/components/schemas/BranchOne"
|
||||
- $ref: "#/components/schemas/BranchAll"
|
||||
- $ref: "#/components/schemas/Identity"
|
||||
- $ref: "#/components/schemas/Postgresql"
|
||||
- $ref: "#/components/schemas/Http"
|
||||
- $ref: "#/components/schemas/Graphql"
|
||||
discriminator:
|
||||
@@ -173,7 +172,6 @@ components:
|
||||
branchone: "#/components/schemas/BranchOne"
|
||||
branchall: "#/components/schemas/BranchAll"
|
||||
identity: "#/components/schemas/Identity"
|
||||
postgresql: "#/components/schemas/Postgresql"
|
||||
http: "#/components/schemas/Http"
|
||||
graphql: "#/components/schemas/Graphql"
|
||||
|
||||
@@ -346,16 +344,6 @@ components:
|
||||
required:
|
||||
- type
|
||||
|
||||
Postgresql:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- postgresql
|
||||
required:
|
||||
- type
|
||||
|
||||
Http:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
Reference in New Issue
Block a user