diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index 01a1f4fa97..b509064f29 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 7cc5a56902..d861034187 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -51,8 +51,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu - + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 8dad6651f8..ed4fd7810b 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu + features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index cf95bcd108..72eaf99004 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -95,7 +95,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu + features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -157,7 +157,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 1ef3d32b99..20a30e936f 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -53,8 +53,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu - + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.gitignore b/.gitignore index 60c80becbf..b157a1679a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ CaddyfileRemoteMalo .direnv .vscode .dev-docker-wrapper* +backend/.minio-data diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c06a2031ab..d050af109b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -12792,6 +12792,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-language" version = "0.1.5" @@ -13878,6 +13888,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", + "tar", "tempfile", "thiserror 2.0.12", "tikv-jemalloc-ctl", @@ -13998,6 +14009,18 @@ dependencies = [ "windmill-parser", ] +[[package]] +name = "windmill-parser-java" +version = "1.478.1" +dependencies = [ + "anyhow", + "serde_json", + "tree-sitter", + "tree-sitter-java", + "wasm-bindgen", + "windmill-parser", +] + [[package]] name = "windmill-parser-nu" version = "1.478.1" @@ -14112,6 +14135,7 @@ dependencies = [ "windmill-parser-csharp", "windmill-parser-go", "windmill-parser-graphql", + "windmill-parser-java", "windmill-parser-nu", "windmill-parser-php", "windmill-parser-py", @@ -14236,17 +14260,20 @@ dependencies = [ "tokio-postgres 0.7.13", "tokio-util", "tracing", + "url", "urlencoding", "uuid", "winapi", "windmill-audit", "windmill-common", "windmill-git-sync", + "windmill-macros", "windmill-parser", "windmill-parser-bash", "windmill-parser-csharp", "windmill-parser-go", "windmill-parser-graphql", + "windmill-parser-java", "windmill-parser-nu", "windmill-parser-php", "windmill-parser-py", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f45590e562..56ad06234e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -23,6 +23,7 @@ members = [ "./parsers/windmill-parser-rust", "./parsers/windmill-parser-csharp", "./parsers/windmill-parser-nu", + "./parsers/windmill-parser-java", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", @@ -62,31 +63,34 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] -deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"] kafka = ["windmill-api/kafka"] nats = ["windmill-api/nats"] otel = ["windmill-common/otel", "windmill-worker/otel"] dind = ["windmill-worker/dind"] -php = ["windmill-worker/php"] -rust = ["windmill-worker/rust"] -mysql = ["windmill-worker/mysql"] -oracledb = ["windmill-worker/oracledb"] -mssql = ["windmill-worker/mssql"] -bigquery = ["windmill-worker/bigquery"] websocket = ["windmill-api/websocket"] postgres_trigger = ["windmill-api/postgres_trigger"] mqtt_trigger = ["windmill-api/mqtt_trigger"] sqs_trigger = ["windmill-api/sqs_trigger"] -python = ["windmill-worker/python"] smtp = ["windmill-api/smtp", "windmill-common/smtp"] -csharp = ["windmill-worker/csharp"] -nu = ["windmill-worker/nu"] license = ["windmill-api/license"] oauth2 = ["windmill-api/oauth2"] http_trigger = ["windmill-api/http_trigger"] zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] +# Languages +python = ["windmill-worker/python"] +deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"] +rust = ["windmill-worker/rust"] +mysql = ["windmill-worker/mysql"] +oracledb = ["windmill-worker/oracledb"] +mssql = ["windmill-worker/mssql"] +bigquery = ["windmill-worker/bigquery"] +php = ["windmill-worker/php"] +csharp = ["windmill-worker/csharp"] +nu = ["windmill-worker/nu"] +java = ["windmill-worker/java"] +all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] [dependencies] @@ -158,6 +162,7 @@ windmill-parser-go = { path = "./parsers/windmill-parser-go" } windmill-parser-rust = { path = "./parsers/windmill-parser-rust" } windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" } windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" } +windmill-parser-java = { path = "./parsers/windmill-parser-java" } windmill-parser-nu = { path = "./parsers/windmill-parser-nu" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } @@ -363,5 +368,6 @@ yaml-rust = "0.4.5" tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } tree-sitter = { version = "0.23.0", features = [] } tree-sitter-c-sharp = "0.23.0" +tree-sitter-java = "0.23.0" oracle = { version = "0.6.3", features = ["chrono"] } rumqttc = { version = "0.24.0", features = ["use-native-tls"]} diff --git a/backend/migrations/20250304181111_add_java.down.sql b/backend/migrations/20250304181111_add_java.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250304181111_add_java.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250304181111_add_java.up.sql b/backend/migrations/20250304181111_add_java.up.sql new file mode 100644 index 0000000000..57256afe18 --- /dev/null +++ b/backend/migrations/20250304181111_add_java.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'java'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["java"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu"]}'::jsonb AND NOT config->'worker_tags' @> '"java"'::jsonb; diff --git a/backend/parsers/windmill-parser-java/Cargo.toml b/backend/parsers/windmill-parser-java/Cargo.toml new file mode 100644 index 0000000000..19809e3120 --- /dev/null +++ b/backend/parsers/windmill-parser-java/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-parser-java" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_java" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +tree-sitter.workspace = true +tree-sitter-java.workspace = true +anyhow.workspace = true +wasm-bindgen.workspace = true +serde_json.workspace = true +# convert_case.workspace = true +# lazy_static.workspace = true +# regex.workspace = true + diff --git a/backend/parsers/windmill-parser-java/src/lib.rs b/backend/parsers/windmill-parser-java/src/lib.rs new file mode 100644 index 0000000000..93aa348cec --- /dev/null +++ b/backend/parsers/windmill-parser-java/src/lib.rs @@ -0,0 +1,446 @@ +#![cfg_attr(target_arch = "wasm32", feature(c_variadic))] + +#[cfg(target_arch = "wasm32")] +pub mod wasm_libc; + +use anyhow::anyhow; +use anyhow::bail; +use serde_json::Value; +use tree_sitter::Node; +use windmill_parser::Arg; +use windmill_parser::MainArgSignature; +use windmill_parser::Typ; + +#[derive(Debug)] +pub struct JavaMainSigMeta { + pub is_public: bool, + pub returns_void: bool, + pub class_name: Option, + pub main_sig: MainArgSignature, +} + +pub fn parse_java_sig_meta(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_java::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting Java as language: {e}"))?; + + // Parse code + let tree = parser + .parse(code, None) + .ok_or(anyhow!("Failed to parse code"))?; + let root_node = tree.root_node(); + + // Traverse the AST to find the Main method signature + let main_sig = find_main_signature(root_node, code); + let no_main_func = Some(main_sig.is_none()); + let mut is_public = false; + let mut returns_void = false; + let mut class_name = None; + + let mut args = vec![]; + if let Some((sig, name)) = main_sig { + class_name = name; + for sig_node in sig.children(&mut sig.walk()) { + if sig_node.kind() == "modifier" && sig_node.utf8_text(code.as_bytes())? == "public" { + is_public = true; + } + } + if let Some(return_type) = sig.child_by_field_name("type") { + let return_type = return_type.utf8_text(code.as_bytes())?; + + if return_type == "void" { + returns_void = true; + } + } + if let Some(param_list) = sig.child_by_field_name("parameters") { + for p_list_node in param_list.children(&mut param_list.walk()) { + if p_list_node.kind() == "formal_parameter" { + let (otyp, typ, name, default) = parse_java_typ(p_list_node, code)?; + args.push(Arg { + name, + otyp, + typ, + has_default: default.is_some(), + default, + oidx: None, + }); + } + } + } + } + + let main_sig = MainArgSignature { + star_args: false, + star_kwargs: false, + args, + has_preprocessor: None, + no_main_func, + }; + + Ok(JavaMainSigMeta { returns_void, class_name, main_sig, is_public }) +} + +pub fn parse_java_signature(code: &str) -> anyhow::Result { + Ok(parse_java_sig_meta(code)?.main_sig) +} + +fn find_typ<'a>(typ_node: Node<'a>, code: &str) -> anyhow::Result<(Typ, Option)> { + let null = Some(serde_json::Value::Null); + let res = match typ_node.kind() { + #[rustfmt::skip] + "type_identifier" => { + match typ_node.utf8_text(code.as_bytes()) { + Ok("String") => (Typ::Str(None), null), + Ok("Byte") => (Typ::Bytes, null), + Ok("Short") => (Typ::Int, null), + Ok("Integer") => (Typ::Int, null), + Ok("Long") => (Typ::Int, null), + Ok("Float") => (Typ::Float, null), + Ok("Double") => (Typ::Float, null), + Ok("Boolean") => (Typ::Bool, null), + Ok("Character") => (Typ::Str(None), null), + Ok("Object") => (Typ::Object(vec![]),null), // TODO: Complete the object type + Ok(s) => bail!("Unknown type `{s}`"), + Err(e) => bail!("Error getting type name: {}", e), + } + } + #[rustfmt::skip] + "integral_type" => { + match typ_node.utf8_text(code.as_bytes()) { + Ok("byte") => (Typ::Bytes, None), + Ok("short") => (Typ::Int, None), + Ok("int") => (Typ::Int, None), + Ok("long") => (Typ::Int, None), + Ok("char") => (Typ::Str(None), None), + Ok(s) => bail!("Unknown type `{s}`"), + Err(e) => bail!("Error getting type name: {}", e), + } + } + "floating_point_type" => { + match typ_node.utf8_text(code.as_bytes()) { + Ok("float") => (Typ::Float, None), + Ok("double") => (Typ::Float, None), + Ok(s) => bail!("Unknown type `{s}`"), + Err(e) => bail!("Error getting type name: {}", e), + } + } + "boolean_type" => { + match typ_node.utf8_text(code.as_bytes()) { + Ok("boolean") => (Typ::Bool, None), + Ok(s) => bail!("Unknown type `{s}`"), + Err(e) => bail!("Error getting type name: {}", e), + } + } + "array_type" => { + let new_typ_node = typ_node + .named_child(0) + .ok_or(anyhow!("Failed to find inner type of array type"))?; + (Typ::List(Box::new(find_typ(new_typ_node, code)?.0)), null) + } + wc => bail!( + "Unexpected Java type node kind: {} for '{}'. This type is not handled by Windmill, please open an issue if this seems to be an error", + wc, + typ_node.utf8_text(code.as_bytes())? + ), + + }; + Ok(res) +} + +fn parse_java_typ<'a>( + param_node: Node<'a>, + code: &str, +) -> anyhow::Result<(Option, Typ, String, Option)> { + let name = param_node + .child_by_field_name("name") + .and_then(|n| n.utf8_text(code.as_bytes()).ok()) + .unwrap_or(""); + let otyp_node = param_node.child_by_field_name("type"); + let otyp = otyp_node + .and_then(|n| n.utf8_text(code.as_bytes()).ok()) + .map(|s| s.to_string()); + + let (typ, default) = find_typ( + otyp_node.ok_or(anyhow!( + "Internal error: Failed to get child by field name 'type'" + ))?, + code, + )?; + + Ok((otyp, typ, name.to_string(), default)) +} + +// Function to find the Main method's signature +fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> Option<(Node<'a>, Option)> { + let mut cursor = root_node.walk(); + for x in root_node.children(&mut cursor) { + if x.kind() == "class_declaration" { + let class_name = x + .child_by_field_name("name") + .and_then(|n| n.utf8_text(code.as_bytes()).ok().map(|s| s.to_string())); + for c in x.children(&mut x.walk()) { + if c.kind() == "class_body" { + for w in c.children(&mut c.walk()) { + if w.kind() == "method_declaration" { + for child in w.children(&mut w.walk()) { + if child + .utf8_text(code.as_bytes()) + .map(|name| name == "main") + .unwrap_or(false) + { + return Some((w, class_name)); + } + } + } + } + } + } + } + } + return None; +} + +#[cfg(test)] +mod test { + + use serde_json::json; + + use super::*; + #[test] + fn test_parse_java_return_void() { + let code = r#" +class Main { + public static void main() {} +}"#; + let sig_meta = parse_java_sig_meta(code).unwrap(); + + assert_eq!(sig_meta.class_name, Some("Main".to_string())); + + assert!(sig_meta.returns_void); + } + #[test] + fn test_parse_java_return_object() { + let code = r#" +class Main { + public static Object main() {} +}"#; + let sig_meta = parse_java_sig_meta(code).unwrap(); + + assert_eq!(sig_meta.class_name, Some("Main".to_string())); + assert!(!sig_meta.returns_void); + } + #[test] + fn test_parse_java_primitive_types() { + let code = r#" +class Main { + public static string main(byte a, short b, int c, long d, float e, double f, boolean g, char h) {} +}"#; + let sig_meta = parse_java_sig_meta(code).unwrap(); + + assert_eq!(sig_meta.class_name, Some("Main".to_string())); + + let ret = sig_meta.main_sig; + assert_eq!( + ret.args, + vec![ + Arg { + name: "a".into(), + otyp: Some("byte".into()), + typ: Typ::Bytes, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "b".into(), + otyp: Some("short".into()), + typ: Typ::Int, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "c".into(), + otyp: Some("int".into()), + typ: Typ::Int, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "d".into(), + otyp: Some("long".into()), + typ: Typ::Int, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "e".into(), + otyp: Some("float".into()), + typ: Typ::Float, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "f".into(), + otyp: Some("double".into()), + typ: Typ::Float, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "g".into(), + otyp: Some("boolean".into()), + typ: Typ::Bool, + default: None, + has_default: false, + oidx: None + }, + Arg { + name: "h".into(), + otyp: Some("char".into()), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + ] + ); + } + + #[test] + fn test_parse_java_objects() { + let code = r#" +class Main { + public static string main(Byte a, Short b, Integer c, Long d, Float e, Double f, Boolean g, Character h, Object i) {} +}"#; + let sig_meta = parse_java_sig_meta(code).unwrap(); + + assert_eq!(sig_meta.class_name, Some("Main".to_string())); + + let ret = sig_meta.main_sig; + assert_eq!( + ret.args, + vec![ + Arg { + name: "a".into(), + otyp: Some("Byte".into()), + typ: Typ::Bytes, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "b".into(), + otyp: Some("Short".into()), + typ: Typ::Int, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "c".into(), + otyp: Some("Integer".into()), + typ: Typ::Int, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "d".into(), + otyp: Some("Long".into()), + typ: Typ::Int, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "e".into(), + otyp: Some("Float".into()), + typ: Typ::Float, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "f".into(), + otyp: Some("Double".into()), + typ: Typ::Float, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "g".into(), + otyp: Some("Boolean".into()), + typ: Typ::Bool, + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "h".into(), + otyp: Some("Character".into()), + typ: Typ::Str(None), + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "i".into(), + otyp: Some("Object".into()), + typ: Typ::Object(vec![]), + default: Some(json!(null)), + has_default: true, + oidx: None + }, + ] + ); + } + #[test] + fn test_parse_java_array() { + let code = r#" +class Main { + public static string main(int[] a, Object[] b, String[] c) {} +}"#; + let sig_meta = parse_java_sig_meta(code).unwrap(); + + assert_eq!(sig_meta.class_name, Some("Main".to_string())); + + let ret = sig_meta.main_sig; + assert_eq!( + ret.args, + vec![ + Arg { + name: "a".into(), + otyp: Some("int[]".into()), + typ: Typ::List(Box::new(Typ::Int)), + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "b".into(), + otyp: Some("Object[]".into()), + typ: Typ::List(Box::new(Typ::Object(vec![]))), + default: Some(json!(null)), + has_default: true, + oidx: None + }, + Arg { + name: "c".into(), + otyp: Some("String[]".into()), + typ: Typ::List(Box::new(Typ::Str(None))), + default: Some(json!(null)), + has_default: true, + oidx: None + }, + ] + ); + } +} diff --git a/backend/parsers/windmill-parser-java/src/wasm_libc.rs b/backend/parsers/windmill-parser-java/src/wasm_libc.rs new file mode 100644 index 0000000000..7d260af2e8 --- /dev/null +++ b/backend/parsers/windmill-parser-java/src/wasm_libc.rs @@ -0,0 +1,207 @@ +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; +use std::{ + alloc::{self, Layout}, + ffi::{c_char, c_int, c_void}, + mem::align_of, + ptr, +}; +use wasm_bindgen::prelude::*; + +/* -------------------------------- stdlib.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn abort() { + panic!("Aborted from C"); +} + +macro_rules! console_log { + ($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) }) +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn log(a: &str); +} + +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size); + let buf = alloc::alloc(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + if count == 0 || size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size * count); + let buf = alloc::alloc_zeroed(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void { + if buf.is_null() { + malloc(new_size) + } else if new_size == 0 { + free(buf); + ptr::null_mut() + } else { + let (old_buf, old_layout) = retrieve_layout(buf); + let (new_layout, offset_to_data) = layout_for_size_prepended(new_size); + let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size()); + store_layout(new_buf, new_layout, offset_to_data) + } +} + +#[no_mangle] +pub unsafe extern "C" fn free(buf: *mut c_void) { + if buf.is_null() { + return; + } + let (buf, layout) = retrieve_layout(buf); + alloc::dealloc(buf, layout); +} + +// In all these allocations, we store the layout before the data for later retrieval. +// This is because we need to know the layout when deallocating the memory. +// Here are some helper methods for that: + +/// Given a pointer to the data, retrieve the layout and the pointer to the layout. +unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) { + let (_, layout_offset) = Layout::new::() + .extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap()) + .unwrap(); + + let buf = (buf as *mut u8).offset(-(layout_offset as isize)); + let layout = *(buf as *mut Layout); + + (buf, layout) +} + +/// Calculate a layout for a given size with space for storing a layout at the start. +/// Returns the layout and the offset to the data. +fn layout_for_size_prepended(size: usize) -> (Layout, usize) { + Layout::new::() + .extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap()) + .unwrap() +} + +/// Store a layout in the pointer, returning a pointer to where the data should be stored. +unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void { + *(buf as *mut Layout) = layout; + (buf as *mut u8).offset(offset_to_data as isize) as *mut c_void +} + +/* -------------------------------- string.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int { + let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n); + let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n); + + for (a, b) in s1.iter().zip(s2.iter()) { + if *a != *b || *a == 0 { + return (*a as i32) - (*b as i32); + } + } + + 0 +} + +/* -------------------------------- wctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn iswspace(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_whitespace()) +} + +#[no_mangle] +pub unsafe extern "C" fn iswalnum(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric()) +} + +/* --------------------------------- time.h --------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn clock() -> u64 { + panic!("clock is not supported"); +} + +/* --------------------------------- ctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn isprint(c: c_int) -> bool { + c >= 32 && c <= 126 +} + +/* --------------------------------- stdio.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int { + panic!("fprintf is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int { + panic!("fputs is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int { + panic!("fputc is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void { + panic!("fdopen is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int { + panic!("fclose is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fwrite( + _ptr: *const c_void, + _size: usize, + _nmemb: usize, + _stream: *mut c_void, +) -> usize { + panic!("fwrite is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn vsnprintf( + _buf: *mut c_char, + _size: usize, + _format: *const c_char, + _args: ... +) -> c_int { + panic!("vsnprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) { + panic!("clock_gettime is not supported"); +} + +// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... ); +#[no_mangle] +pub extern "C" fn snprintf() { + panic!("snprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) { + panic!("oh no"); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index fe450c32b4..5b1c884143 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -28,6 +28,7 @@ graphql-parser = [ "dep:windmill-parser-graphql"] ansible-parser = [ "dep:windmill-parser-yaml"] csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] +java-parser = [ "dep:windmill-parser-java"] [dependencies] anyhow.workspace = true @@ -43,6 +44,7 @@ windmill-parser-rust = { workspace = true, optional = true } windmill-parser-yaml = { workspace = true, optional = true } windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } +windmill-parser-java = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true getrandom = { workspace = true, features = ["js"] } diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs-cli.sh b/backend/parsers/windmill-parser-wasm/build-pkgs-cli.sh index 34c36e8820..33c6701e52 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs-cli.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs-cli.sh @@ -70,6 +70,10 @@ OUT_DIR="../../../cli/wasm/nu" mkdir -p $OUT_DIR wasm-pack build --release --target deno --out-dir $OUT_DIR --features "nu-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -# sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json rm $OUT_DIR/.gitignore +#-# Java +OUT_DIR="../../../cli/wasm/java" +mkdir -p $OUT_DIR +CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target deno --out-dir $OUT_DIR --features "java-parser" +rm $OUT_DIR/.gitignore diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh index 85de323a90..109a48e9b3 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh @@ -61,3 +61,8 @@ OUT_DIR="pkg-nu" wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json + +#-# Java (needs some more stuff to compile C tree sitter into wasm) +OUT_DIR="pkg-java" +CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser" +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs.sh b/backend/parsers/windmill-parser-wasm/build-pkgs.sh index e2f0b6b471..0d492e9dcb 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs.sh @@ -60,3 +60,7 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser" -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json +#-# Java (needs some more stuff to compile C tree sitter into wasm) +OUT_DIR="pkg-java" +CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser" +sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/build.sh b/backend/parsers/windmill-parser-wasm/build.sh deleted file mode 100755 index c843d2b445..0000000000 --- a/backend/parsers/windmill-parser-wasm/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -eou pipefail - -# Flags are needed to build correctly with C# -# NOTE: Currently does not work with nix nor nixos -CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" deno task wasmbuild --inline --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features - - diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2ca8f2f2b0..4dbe849e34 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -155,3 +155,11 @@ pub fn parse_csharp(code: &str) -> String { pub fn parse_nu(code: &str) -> String { wrap_sig(windmill_parser_nu::parse_nu_signature(code)) } + +#[cfg(feature = "java-parser")] +#[wasm_bindgen] +pub fn parse_java(code: &str) -> String { + wrap_sig(windmill_parser_java::parse_java_signature(code)) +} + +// KJQXZ diff --git a/backend/src/main.rs b/backend/src/main.rs index a3ec60f846..aec045c8f3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,7 +9,8 @@ use anyhow::Context; use monitor::{ load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, - reload_instance_python_version_setting, reload_nuget_config_setting, + reload_instance_python_version_setting, reload_maven_repos_setting, + reload_no_default_maven_setting, reload_nuget_config_setting, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, }; @@ -37,11 +38,12 @@ use windmill_common::{ EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, - TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, + NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -66,9 +68,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, - NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, - PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, - TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR, + TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -796,6 +798,12 @@ Windmill Community Edition {GIT_VERSION} NUGET_CONFIG_SETTING => { reload_nuget_config_setting(&db).await }, + MAVEN_REPOS_SETTING => { + reload_maven_repos_setting(&db).await + }, + NO_DEFAULT_MAVEN_SETTING => { + reload_no_default_maven_setting(&db).await + }, KEEP_JOB_DIR_SETTING => { load_keep_job_dir(&db).await; }, @@ -1103,6 +1111,8 @@ pub async fn run_workers( NU_CACHE_DIR, HUB_CACHE_DIR, POWERSHELL_CACHE_DIR, + JAVA_CACHE_DIR, + TAR_JAVA_CACHE_DIR, // KJQXZ ] { DirBuilder::new() .recursive(true) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index c94651345d..1209b2aca3 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -52,9 +52,7 @@ use windmill_common::{ }; use windmill_queue::{cancel_job, MiniPulledJob}; use windmill_worker::{ - create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, - SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, - NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY }; #[cfg(feature = "parquet")] @@ -186,6 +184,8 @@ pub async fn initial_load( reload_bunfig_install_scopes_setting(&db).await; reload_instance_python_version_setting(&db).await; reload_nuget_config_setting(&db).await; + reload_maven_repos_setting(&db).await; + reload_no_default_maven_setting(&db).await; } } @@ -969,6 +969,20 @@ pub async fn reload_nuget_config_setting(db: &DB) { ) .await; } +pub async fn reload_maven_repos_setting(db: &DB) { + reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()) + .await; +} +pub async fn reload_no_default_maven_setting(db: &DB) { + let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await; + match value { + Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed), + Err(e) => { + tracing::error!("Error loading no default maven repository: {e:#}"); + } + _ => (), + }; +} pub async fn reload_retention_period_setting(db: &DB) { if let Err(e) = reload_setting( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 2c1e47f2af..f942a8ee6e 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1988,6 +1988,51 @@ def main [ assert_eq!(result, serde_json::json!(0)); } +#[cfg(feature = "java")] +#[sqlx::test(fixtures("base"))] +async fn test_java_job(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +public class Main { + public static Object main( + // Primitive + int a, + float b, + // Objects + Integer age, + Float d + ){ + return "hello world"; + } +} + +"# + .to_owned(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Java, + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + })) + .arg("a", json!(3)) + .arg("b", json!(3.0)) + .arg("age", json!(30)) + .arg("d", json!(3.0)) + .run_until_complete(&db, port) + .await; + assert_eq!(job.json_result(), Some(json!("hello world"))); +} + #[sqlx::test(fixtures("base"))] async fn test_python_job(db: Pool) { initialize_tracing().await; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index acd899e985..c89d759c41 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13529,7 +13529,9 @@ components: rust, ansible, csharp, - nu + nu, + java + # KJQXZ ] Preview: diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index f370bf33fe..4861490ad0 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -624,17 +624,20 @@ async fn create_script_internal<'c>( .unwrap_or(json!({})); let lock = if ns.codebase.is_some() { Some(String::new()) - } else if !(ns.language == ScriptLang::Python3 - || ns.language == ScriptLang::Go - || ns.language == ScriptLang::Bun - || ns.language == ScriptLang::Bunnative - || ns.language == ScriptLang::Deno - || ns.language == ScriptLang::Rust - || ns.language == ScriptLang::Ansible - || ns.language == ScriptLang::CSharp - || ns.language == ScriptLang::Nu - || ns.language == ScriptLang::Php) - { + } else if !( + ns.language == ScriptLang::Python3 + || ns.language == ScriptLang::Go + || ns.language == ScriptLang::Bun + || ns.language == ScriptLang::Bunnative + || ns.language == ScriptLang::Deno + || ns.language == ScriptLang::Rust + || ns.language == ScriptLang::Ansible + || ns.language == ScriptLang::CSharp + || ns.language == ScriptLang::Nu + || ns.language == ScriptLang::Php + || ns.language == ScriptLang::Java + // KJQXZ + ) { Some(String::new()) } else { ns.lock diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 1e4a689b18..ef5e089bd3 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -375,6 +375,8 @@ pub(crate) async fn tarball_workspace( ScriptLang::CSharp => "cs", ScriptLang::Nu => "nu", ScriptLang::OracleDB => "odb.sql", + ScriptLang::Java => "java", + // KJQXZ }; archive .write_to_archive(&script.content, &format!("{}.{}", script.path, ext)) diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 7ff360b42b..fd370aedcc 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -24,6 +24,7 @@ name = "windmill_common" path = "src/lib.rs" [dependencies] +tar.workspace = true hmac.workspace = true sha2.workspace = true thiserror.workspace = true diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index e0c091a0da..f92eac988c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -11,6 +11,8 @@ pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const NUGET_CONFIG_SETTING: &str = "nuget_config"; +pub const MAVEN_REPOS_SETTING: &str = "maven_repos"; +pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; @@ -61,6 +63,8 @@ pub const ENV_SETTINGS: &[&str] = &[ "NU_PATH", "DENO_PATH", "GO_PATH", + "JAVA_PATH", + // KJQXZ "GOPRIVATE", "GOPROXY", "NETRC", diff --git a/backend/windmill-common/src/schema.rs b/backend/windmill-common/src/schema.rs index 4b97f761ae..201cdbe92f 100644 --- a/backend/windmill-common/src/schema.rs +++ b/backend/windmill-common/src/schema.rs @@ -345,27 +345,15 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool { pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool { let annotation = "schema_validation"; - match lang { - ScriptLang::Nativets | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno => { - find_annotation("//", annotation, code) - } - ScriptLang::Python3 => find_annotation("#", annotation, code), - ScriptLang::Go => find_annotation("#", annotation, code), - ScriptLang::Bash => find_annotation("#", annotation, code), - ScriptLang::Powershell => find_annotation("#", annotation, code), - ScriptLang::Postgresql => find_annotation("--", annotation, code), - ScriptLang::Mysql => find_annotation("--", annotation, code), - ScriptLang::Bigquery => find_annotation("--", annotation, code), - ScriptLang::Snowflake => find_annotation("--", annotation, code), - ScriptLang::Graphql => find_annotation("#", annotation, code), - ScriptLang::Mssql => find_annotation("--", annotation, code), - ScriptLang::OracleDB => find_annotation("--", annotation, code), - ScriptLang::Php => find_annotation("//", annotation, code), - ScriptLang::Rust => find_annotation("//!", annotation, code), - ScriptLang::Ansible => find_annotation("#", annotation, code), - ScriptLang::CSharp => find_annotation("//", annotation, code), - ScriptLang::Nu => find_annotation("#", annotation, code), - } + use ScriptLang::*; + let comment = match lang { + Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//", + Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--", + Rust => "//!", + // KJQXZ + }; + find_annotation(comment, annotation, code) } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index b38598f0b8..ab478b40e0 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -51,6 +51,8 @@ pub enum ScriptLang { Ansible, CSharp, Nu, + Java, + // KJQXZ } impl ScriptLang { @@ -76,6 +78,8 @@ impl ScriptLang { ScriptLang::Ansible => "ansible", ScriptLang::CSharp => "csharp", ScriptLang::Nu => "nu", + ScriptLang::Java => "java", + // KJQXZ } } } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index a6979ad30e..39cfc7771a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -9,7 +9,7 @@ use serde_json::value::RawValue; use std::{ cmp::Reverse, collections::{HashMap, HashSet}, - fs::File, + fs::{self, File}, io::Write, path::{Component, Path, PathBuf}, str::FromStr, @@ -51,6 +51,8 @@ lazy_static::lazy_static! { "ansible".to_string(), "csharp".to_string(), "nu".to_string(), + "java".to_string(), + // KJQXZ "dependency".to_string(), "flow".to_string(), "other".to_string() @@ -402,8 +404,43 @@ pub struct SqlAnnotations { pub struct BashAnnotations { pub docker: bool, } +/// length = 5 +/// value = "foo" +/// output = "foo " +/// 12345 +pub fn pad_string(value: &str, total_length: usize) -> String { + if value.len() >= total_length { + value.to_string() // Return the original string if it's already long enough + } else { + let padding_needed = total_length - value.len(); + format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces + } +} +pub fn copy_dir_recursively(src: &Path, dst: &Path) -> error::Result<()> { + if !dst.exists() { + fs::create_dir_all(dst)?; + } -pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { + tracing::debug!("Copying recursively from {:?} to {:?}", src, dst); + + for entry in fs::read_dir(src)? { + let entry = entry?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + + if src_path.is_dir() && !src_path.is_symlink() { + copy_dir_recursively(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path)?; + } + } + + tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst); + + Ok(()) +} + +pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) } else { @@ -417,12 +454,22 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { use crate::s3_helpers::attempt_fetch_bytes; if let Ok(mut x) = attempt_fetch_bytes(os, _remote_path).await { - if let Err(e) = write_binary_file(bin_path, &mut x) { - tracing::error!("could not write bundle/bin file locally: {e:?}"); - return ( - false, - "error writing bundle/bin file from object store".to_string(), - ); + if is_dir { + if let Err(e) = extract_tar(x, bin_path).await { + tracing::error!("could not write tar archive locally: {e:?}"); + return ( + false, + "error writing tar archive from object store".to_string(), + ); + } + } else { + if let Err(e) = write_binary_file(bin_path, &mut x) { + tracing::error!("could not write bundle/bin file locally: {e:?}"); + return ( + false, + "error writing bundle/bin file from object store".to_string(), + ); + } } tracing::info!("loaded from object store {}", bin_path); return ( @@ -435,6 +482,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { ); } } + let _ = is_dir; (false, "".to_string()) } } @@ -462,6 +510,7 @@ pub async fn save_cache( local_cache_path: &str, _remote_cache_path: &str, origin: &str, + is_dir: bool, ) -> crate::error::Result { let mut _cached_to_s3 = false; #[cfg(all(feature = "enterprise", feature = "parquet"))] @@ -471,11 +520,33 @@ pub async fn save_cache( .clone() { use object_store::path::Path; + let file_to_cache = if is_dir { + let tar_path = format!( + "{ROOT_CACHE_DIR}/tar/{}_tar.tar", + local_cache_path + .split("/") + .last() + .unwrap_or(&uuid::Uuid::new_v4().to_string()) + ); + let tar_file = std::fs::File::create(&tar_path)?; + let mut tar = tar::Builder::new(tar_file); + tar.append_dir_all(".", &origin)?; + let tar_metadata = tokio::fs::metadata(&tar_path).await; + if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 { + tracing::info!("Failed to tar cache: {origin}"); + return Err(error::Error::ExecutionErr(format!( + "Failed to tar cache: {origin}" + ))); + } + tar_path + } else { + origin.to_owned() + }; if let Err(e) = os .put( &Path::from(_remote_cache_path), - std::fs::read(origin)?.into(), + std::fs::read(&file_to_cache)?.into(), ) .await { @@ -485,12 +556,19 @@ pub async fn save_cache( ); } else { _cached_to_s3 = true; + if is_dir { + tokio::fs::remove_dir_all(&file_to_cache).await?; + } } } // if !*CLOUD_HOSTED { if true { - std::fs::copy(origin, local_cache_path)?; + if is_dir { + copy_dir_recursively(&PathBuf::from(origin), &PathBuf::from(local_cache_path))?; + } else { + std::fs::copy(origin, local_cache_path)?; + } Ok(format!( "\nwrote cached binary: {} (backed by EE distributed object store: {_cached_to_s3})\n", local_cache_path @@ -505,6 +583,31 @@ pub async fn save_cache( } } +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { + use std::time::Instant; + + use bytes::Buf; + use tokio::fs::{self}; + + let start: Instant = Instant::now(); + fs::create_dir_all(&folder).await?; + + let mut ar = tar::Archive::new(tar.reader()); + + if let Err(e) = ar.unpack(folder) { + tracing::info!("Failed to untar to {folder}. Error: {:?}", e); + fs::remove_dir_all(&folder).await?; + return Err(error::Error::ExecutionErr(format!( + "Failed to untar tar {folder}" + ))); + } + tracing::info!( + "Finished extracting tar to {folder}. Took {}ms", + start.elapsed().as_millis(), + ); + Ok(()) +} #[cfg(all(feature = "enterprise", feature = "parquet"))] fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::{File, Permissions}; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 291e92c2df..510a33a645 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -30,17 +30,20 @@ python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"] csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] +java = ["dep:windmill-parser-java"] [dependencies] windmill-queue.workspace = true windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. windmill-common = { workspace = true, default-features = false } +windmill-macros.workspace = true windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-go.workspace = true windmill-parser-rust = { workspace = true, optional = true } windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } +windmill-parser-java = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } windmill-parser-yaml.workspace = true windmill-parser-py-imports = { workspace = true, optional = true } @@ -72,6 +75,7 @@ dyn-iter.workspace = true once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true +url.workspace = true deno_telemetry = { workspace = true, optional = true } deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } @@ -127,4 +131,4 @@ deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_runtime = { workspace = true, optional = true } deno_telemetry = { workspace = true, optional = true } -winapi = { workspace = true, optional = true } \ No newline at end of file +winapi = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/run.java.config.proto b/backend/windmill-worker/nsjail/run.java.config.proto new file mode 100644 index 0000000000..032072fb7c --- /dev/null +++ b/backend/windmill-worker/nsjail/run.java.config.proto @@ -0,0 +1,107 @@ +name: "java run script" + +mode: ONCE +hostname: "java" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + + +mount { + src: "{JOB_DIR}/target" + dst: "/tmp/target" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/args.json" + dst: "/tmp/args.json" + is_bind: true +} + +mount { + src: "{JOB_DIR}/result.json" + dst: "/tmp/result.json" + rw: true + is_bind: true +} + +mount { + src: "{CACHE_DIR}" + dst: "{CACHE_DIR}" + is_bind: true + mandatory: false +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +{SHARED_MOUNT} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index c98837ac5d..5699c44932 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -170,6 +170,7 @@ async fn install_galaxy_collections( None, false, &mut Some(occupancy_metrics), + None, ) .await?; @@ -432,6 +433,7 @@ fi job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; read_and_check_result(job_dir).await diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index a496b327b5..85c20fd6fd 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -226,6 +226,7 @@ exit $exit_status job.timeout, true, &mut Some(occupancy_metrics), + None, ) .await?; @@ -569,6 +570,7 @@ pub async fn handle_powershell_job( job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; } @@ -780,6 +782,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index ed6881b4fd..382e07eae4 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -166,6 +166,7 @@ pub async fn gen_bun_lockfile( None, false, occupancy_metrics, + None, ) .await?; } else { @@ -349,6 +350,7 @@ pub async fn install_bun_lockfile( None, false, occupancy_metrics, + None, ) .await? } else { @@ -520,6 +522,7 @@ pub async fn generate_wrapper_mjs( timeout, false, occupancy_metrics, + None, ) .await?; fs::rename( @@ -570,6 +573,7 @@ pub async fn generate_bun_bundle( timeout, false, occupancy_metrics, + None, ) .await?; } else { @@ -594,7 +598,6 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if std::fs::metadata(&bun_cache_path).is_ok() { tracing::info!("loading {bun_cache_path} from cache"); - extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] @@ -733,7 +736,7 @@ pub async fn prebundle_bun_script( ) .await?; - save_cache(&local_path, &remote_path, &origin).await?; + save_cache(&local_path, &remote_path, &origin, false).await?; Ok(()) } @@ -852,7 +855,8 @@ pub async fn handle_bun_job( ) .await; - let (cache, logs) = windmill_common::worker::load_cache(&local_path, &remote_path).await; + let (cache, logs) = + windmill_common::worker::load_cache(&local_path, &remote_path, false).await; (cache, logs, local_path, remote_path) } else { (false, "".to_string(), "".to_string(), "".to_string()) @@ -1181,7 +1185,14 @@ try {{ ) .await?; if !local_path.is_empty() { - match save_cache(&local_path, &remote_path, &format!("{job_dir}/main.js")).await { + match save_cache( + &local_path, + &remote_path, + &format!("{job_dir}/main.js"), + false, + ) + .await + { Err(e) => { let em = format!("could not save {local_path} to bundle cache: {e:?}"); tracing::error!(em) @@ -1421,6 +1432,7 @@ try {{ job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index cd4f26b776..039ff4205b 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -11,6 +11,7 @@ use sha2::Digest; use sqlx::types::Json; use sqlx::{Pool, Postgres}; use tokio::process::Command; +use tokio::sync::{RwLock, Semaphore}; use tokio::{fs::File, io::AsyncReadExt}; #[cfg(feature = "parquet")] @@ -28,7 +29,7 @@ use windmill_common::{ variables::ContextualVariable, }; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use windmill_queue::MiniPulledJob; use std::path::Path; @@ -39,7 +40,10 @@ use windmill_common::{variables, DB}; use tokio::{io::AsyncWriteExt, process::Child, time::Instant}; -use crate::{AuthedClient, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV}; +use crate::{ + AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, + PATH_ENV, +}; pub async fn build_args_map<'a>( job: &'a MiniPulledJob, @@ -946,3 +950,575 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result .build() .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) } + +#[derive(Clone)] +pub struct RequiredDependency { + /// Expected directory of dependency in cache + /// For example: + /// /tmp/windmill/cache/python_311/rich==0.0.0 + /// IMPORTANT!: path should not end with '/' + pub path: String, + /// Name to use for S3 tars + /// If not specified will use top level directory of path. + pub custom_name: Option, + /// Display name + /// Name that will be used for console output and logging + /// If not specified will either use custom_name or top level directory of path. + pub short_name: Option, +} + +pub enum InstallStrategy { + /// Will invoke callback to install single dependency + Single(Arc Result + Send + Sync>), + /// Will try to pull S3 first and will invoke closure to install the rest + AllAtOnce(Arc) -> Result + Send + Sync>), +} +/// # General +/// +/// Languages that compile usually include dependencies in final executable. +/// When dynamic languages do not and runtime dependencies provided separately. +/// +/// This helper implies that the language is dynamic. +/// Python, Ruby, Java are dynamic and they can use this helper. +/// +/// # Features +/// +/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3 +/// It has atomic success file, allowing to distinguish failed installations from succesfull. +/// +/// Besides that it provides console output and does logging. +/// +/// # Usage +/// +/// Most important arguments in this helper are `deps` and `install_fn` +/// +/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script. +/// You don't know which are actually installed and which are not. +/// +/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context. +/// +/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3 +/// and if it does not work either, it will invoke `install_fn` closure. +/// Closure arguments has dependency name as well as it`s expected path in cache. +/// Closure should return Command that will install dependency to asked place. +pub async fn par_install_language_dependencies<'a>( + deps: Vec, + language_name: &'a str, + installer_executable_name: &'a str, + platform_agnostic: bool, + concurrent_downloads: usize, + stdout_on_err: bool, + install_fn: InstallStrategy, + postinstall_cb: impl AsyncFn(Vec) -> Result<(), error::Error>, + job_id: &'a Uuid, + w_id: &'a str, + worker_name: &'a str, + db: &sqlx::Pool, +) -> anyhow::Result<()> { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + let _ = (platform_agnostic, language_name); + + let total_time = std::time::Instant::now(); + + // Total to install + let mut not_installed = vec![]; + let total_to_install; + // let mut not_installed = vec![]; + let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); + // Append logs with line like this: + // [9/21] + requests==2.32.3 << (S3) | in 57ms + #[allow(unused_assignments)] + async fn print_success( + mut s3_pull: bool, + mut s3_push: bool, + job_id: &Uuid, + w_id: &str, + req: &str, + req_tl: usize, + counter_arc: Arc>, + total_to_install: usize, + instant: std::time::Instant, + db: Pool, + ) { + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + { + (s3_pull, s3_push) = (false, false); + } + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + .read() + .await + .is_none() + { + (s3_pull, s3_push) = (false, false); + } + + let mut counter = counter_arc.lock().await; + *counter += 1; + + windmill_queue::append_logs( + job_id, + w_id, + format!( + "\n{}+ {}{}{}| in {}ms", + windmill_common::worker::pad_string( + &format!("[{}/{total_to_install}]", counter), + 9 + ), + // Because we want to align to max len [999/999] we take 9 + // 123456789 + windmill_common::worker::pad_string(&req, req_tl + 1), + // Margin to the right ^ + if s3_pull { "<< (S3) " } else { "" }, + if s3_push { " > (S3) " } else { "" }, + instant.elapsed().as_millis(), + ), + db, + ) + .await; + // Drop lock, so next print success can fire + } + + let mut name_tl = 0; + struct NotInstalledDependency { + path: String, + custom_name: Option, + short_name: Option, + display_name: String, + } + { + let mut to_be_installed_is_used = false; + for RequiredDependency { + path, // + custom_name, + short_name, + } in deps.into_iter() + { + if path.ends_with("/") { + anyhow::bail!("Internal error: path should not end with '/'") + } + let display_name = short_name + .as_ref() + .or(custom_name.as_ref()) + .or(path.split("/").last().map(|e| e.to_owned()).as_ref()) + .unwrap_or_else(|| { + tracing::warn!( + workspace_id = %w_id, + job_id = %job_id, + "failed to parse top level directory name for {path}, fallback to full path.", + ); + + &path + }) + .to_owned(); + { + // Later will help us align text in log console + if display_name.len() > name_tl { + name_tl = display_name.len(); + } + } + // Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill + if tokio::fs::metadata(path.clone() + ".valid.windmill") + .await + .is_err() + { + if !to_be_installed_is_used { + windmill_queue::append_logs( + job_id, + w_id, + format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"), + db.clone(), + ) + .await; + to_be_installed_is_used = true; + } + windmill_queue::append_logs( + job_id, + w_id, + format!("- {display_name}\n"), + db.clone(), + ) + .await; + not_installed.push(NotInstalledDependency { + path, + custom_name, + short_name, + display_name, + }); + } + } + } + total_to_install = not_installed.len(); + if total_to_install == 0 { + return Ok(()); + } + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let is_not_pro = !matches!( + windmill_common::ee::get_license_plan().await, + windmill_common::ee::LicensePlan::Pro + ); + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) { + windmill_queue::append_logs( + job_id, + w_id, + format!("\nLooking for packages on S3:\n"), + db.clone(), + ) + .await; + } + + // Parallelism level (N) + let parallel_limit = // Semaphore will panic if value less then 1 + concurrent_downloads.clamp(1, 30); + + tracing::info!( + workspace_id = %w_id, + "Install parallel limit: {}, job: {}", + parallel_limit, + job_id + ); + + let mut handles = vec![]; + let semaphore = Arc::new(Semaphore::new(parallel_limit)); + let not_pulled = Arc::new(RwLock::new(vec![])); + // let mut handles = Vec::with_capacity(total_to_install); + for NotInstalledDependency { + // + path, + custom_name, + short_name, + display_name, + } in not_installed + { + let permit = semaphore.clone().acquire_owned().await; // Acquire a permit + + if let Err(_) = permit { + tracing::error!( + workspace_id = %w_id, + "Cannot acquire permit on semaphore, that can only mean that semaphore has been closed." + ); + break; + } + + let permit = permit.unwrap(); + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + let s3_pull_future = if is_not_pro { + if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + .read() + .await + .clone() + { + Some(crate::global_cache::pull_from_tar( + os, + path.clone(), + language_name.to_owned(), + custom_name.clone(), + platform_agnostic, + )) + } else { + None + } + } else { + None + }; + let child = { + if let InstallStrategy::Single(ref callback, ..) = install_fn { + let cmd = callback(RequiredDependency { + path: path.clone(), + custom_name: custom_name.clone(), + short_name: short_name.clone(), + })?; + tracing::debug!("{:?}", &cmd); + Some(start_child_process(cmd, &installer_executable_name).await?) + } else { + None + } + }; + + let ( + worker_name_2, + path_2, + display_name_2, + custom_name, + job_id_2, + w_id_2, + db_2, + counter_arc, + language_name, + installer_executable_name, + not_pulled, + ) = ( + worker_name.to_owned(), + path.clone(), + display_name.clone(), + custom_name.clone(), + job_id.clone(), + w_id.to_owned(), + db.clone(), + counter_arc.clone(), + language_name.to_owned(), + installer_executable_name.to_owned(), + not_pulled.clone(), + ); + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] + let _ = language_name; + + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + let _permit = permit; + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(s3_pull_future) = s3_pull_future { + if let Err(e) = s3_pull_future.await { + tracing::info!( + workspace_id = %w_id_2, + "No tarball was found for {:?} on S3 or different problem occured {job_id_2}:\n{e}", + &custom_name.clone().unwrap_or(path) + ); + } else { + // TODO: Refactor + // Create a file to indicate that installation was successfull + let valid_path = path_2.clone() + ".valid.windmill"; + // This is atomic operation, meaning, that it either completes and dependency is valid, + // or it does not and dependency is invalid and will be reinstalled next run + if let Err(e) = File::create(&valid_path).await { + tracing::error!( + workspace_id = %w_id_2, + job_id = %job_id_2, + "Failed to create {}!\n{e}\n + This file needed for jobs to function", + valid_path + ); + }; + print_success( + true, + false, + &job_id_2, + &w_id_2, + &display_name_2, + name_tl, + counter_arc, + total_to_install, + start, + db_2, + ) + .await; + return; + } + } + + let Some(child) = child else { + let mut lock = not_pulled.write().await; + lock.push(RequiredDependency { + path: path_2.clone(), + custom_name: custom_name.clone(), + short_name: short_name.clone(), + }); + return; + }; + if let Err(e) = crate::handle_child::handle_child( + &job_id_2, + &db_2, + // TODO: Return mem_peak + &mut 0, + // TODO: Return canceld_by_ref + &mut None, + child, + !*DISABLE_NSJAIL, + &worker_name_2, + &w_id_2, + &installer_executable_name, + None, + false, + &mut None, + None, + ) + .await + { + windmill_queue::append_logs( + &job_id_2, + &w_id_2, + format!("error while installing {}: {e:?}", &display_name_2), + db_2.clone(), + ) + .await; + } else { + // if let Some(cb) = postinstall_cb { + // if let Err(e) = cb(vec![RequiredDependency { + // path: path_2.clone(), + // custom_name: custom_name.clone(), + // short_name: short_name.clone(), + // }]) + // .await + // { + // tracing::error!( + // workspace_id = %w_id_2, + // job_id = %job_id_2, + // "Postinstall callback failed!\n{e}\n + // This might affect execution", + // ); + // } + // } + print_success( + false, + true, + &job_id_2, + &w_id_2, + &display_name_2, + name_tl, + counter_arc, + total_to_install, + start, + db_2, + ) + .await; + // TODO: Refactor + // Create a file to indicate that installation was successfull + let valid_path = path_2.clone() + ".valid.windmill"; + // This is atomic operation, meaning, that it either completes and dependency is valid, + // or it does not and dependency is invalid and will be reinstalled next run + if let Err(e) = File::create(&valid_path).await { + tracing::error!( + workspace_id = %w_id_2, + job_id = %job_id_2, + "Failed to create {}!\n{e}\n + This file needed for jobs to function", + valid_path + ); + }; + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + .read() + .await + .clone() + { + tokio::spawn(async move { + if let Err(e) = crate::global_cache::build_tar_and_push( + os, + path_2, + language_name, + custom_name, + platform_agnostic, + ) + .await + { + tracing::warn!("failed to build tar and push: {e:?}"); + } + }); + } + } + } + }); + handles.push(handle); + } + + for handle in handles { + if let Err(e) = handle.await { + tracing::error!("Error joining handles: {e:?}"); + } + } + if !not_pulled.read().await.is_empty() { + if let InstallStrategy::AllAtOnce(ref callback, ..) = install_fn { + let not_pulled_copy = not_pulled.read().await.clone(); + windmill_queue::append_logs( + job_id, + w_id, + format!("\n\nFetching {} packages...\n", not_pulled_copy.len()), + db.clone(), + ) + .await; + let cmd = callback(not_pulled_copy.clone())?; + tracing::debug!("{:?}", &cmd); + let child = start_child_process(cmd, &installer_executable_name).await?; + let mut buf = "".to_owned(); + let pipe_stdout = if stdout_on_err { Some(&mut buf) } else { None }; + if let Err(e) = crate::handle_child::handle_child( + // &job_id, + &Uuid::nil(), + &db, + // TODO: Return mem_peak + &mut 0, + // TODO: Return canceld_by_ref + &mut None, + child, + !*DISABLE_NSJAIL, + &worker_name, + &w_id, + &installer_executable_name, + None, + false, + &mut None, + pipe_stdout, + ) + .await + { + bail!(format!( + "error while installing dependencies: {e:?}\n{}", + buf + )); + } + { + postinstall_cb(not_pulled_copy.clone()).await?; + } + for RequiredDependency { path, custom_name: _custom_name, .. } in + not_pulled_copy.into_iter() + { + // TODO: Refactor + // Create a file to indicate that installation was successfull + let valid_path = path.clone() + ".valid.windmill"; + // This is atomic operation, meaning, that it either completes and dependency is valid, + // or it does not and dependency is invalid and will be reinstalled next run + if let Err(e) = File::create(&valid_path).await { + tracing::error!( + workspace_id = %w_id, + job_id = %job_id, + "Failed to create {}!\n{e}\n + This file needed for jobs to function", + valid_path + ); + }; + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + .read() + .await + .clone() + { + let language_name = language_name.to_owned(); + tokio::spawn(async move { + if let Err(e) = crate::global_cache::build_tar_and_push( + os, + path, + language_name, + _custom_name, + platform_agnostic, + ) + .await + { + tracing::warn!("failed to build tar and push: {e:?}"); + } + }); + } + } + } + } + } + { + let total_time = total_time.elapsed().as_millis(); + windmill_queue::append_logs( + &job_id, + w_id, + format!( + "\nDone. Time spent on installation phase: {}ms\n", + total_time + ), + db, + ) + .await; + } + Ok(()) +} diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 9acb32b684..f30d653ff5 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -126,6 +126,7 @@ pub async fn generate_nuget_lockfile( None, false, &mut Some(occupancy_metrics), + None, ) .await?; @@ -381,6 +382,7 @@ async fn build_cs_proj( None, false, &mut Some(occupancy_metrics), + None, ) .await?; append_logs(job_id, w_id, "\n\n", db).await; @@ -400,6 +402,7 @@ async fn build_cs_proj( &bin_path, &format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"), &target, + false, ) .await { @@ -474,7 +477,8 @@ pub async fn handle_csharp_job( let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await; + let (cache, cache_logs) = + windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { #[cfg(unix)] @@ -634,6 +638,7 @@ pub async fn handle_csharp_job( job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; read_result(job_dir).await diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 2b93c86ab4..3e7f7f2bde 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -156,6 +156,7 @@ pub async fn generate_deno_lock( None, false, occupancy_metrics, + None, ) .await?; } else { @@ -414,6 +415,7 @@ try {{ job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 23ce255615..6d0159d64e 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -17,8 +17,9 @@ pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std: pub async fn build_tar_and_push( s3_client: Arc, folder: String, - // python_311 - python_xyz: String, + lang: String, + custom_folder_name: Option, + platform_agnostic: bool, ) -> error::Result<()> { use object_store::path::Path; @@ -28,9 +29,13 @@ pub async fn build_tar_and_push( let start = Instant::now(); // e.g. tiny==1.0.0 - let folder_name = folder.split("/").last().unwrap(); + let folder_name = if let Some(name) = custom_folder_name { + name + } else { + folder.split("/").last().unwrap().to_owned() + }; - let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz); + let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang); let tar_path = format!("{prefix}/{folder_name}_tar.tar",); let tar_file = std::fs::File::create(&tar_path)?; @@ -51,7 +56,10 @@ pub async fn build_tar_and_push( // })?; if let Err(e) = s3_client .put( - &Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")), + &Path::from(format!( + "/tar/{}/{lang}/{folder_name}.tar", + if platform_agnostic { "" } else { TARGET } + )), std::fs::read(&tar_path)?.into(), ) .await @@ -79,18 +87,26 @@ pub async fn build_tar_and_push( pub async fn pull_from_tar( client: Arc, folder: String, - // python_311 - python_xyz: String, + lang: String, + custom_folder_name: Option, + platform_agnostic: bool, ) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; - let folder_name = folder.split("/").last().unwrap(); + let folder_name = if let Some(name) = custom_folder_name { + name + } else { + folder.split("/").last().unwrap().to_owned() + }; - tracing::info!("Attempting to pull piptar {folder_name} from bucket"); + tracing::info!("Attempting to pull tar {folder_name} from bucket"); let start = Instant::now(); - let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar"); + let tar_path = format!( + "tar/{}/{lang}/{folder_name}.tar", + if platform_agnostic { "" } else { TARGET } + ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; extract_tar(bytes, &folder).map_err(|e| { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 43e88c59aa..959845f641 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -65,7 +65,8 @@ pub async fn handle_go_job( )); let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await; + let (cache, cache_logs) = + windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; let (skip_go_mod, skip_tidy) = if cache { (true, true) @@ -212,6 +213,7 @@ func Run(req Req) (interface{{}}, error){{ None, false, &mut Some(occupation_metrics), + None, ) .await?; @@ -219,6 +221,7 @@ func Run(req Req) (interface{{}}, error){{ &bin_path, &format!("{GO_OBJECT_STORE_PREFIX}{hash}"), &format!("{job_dir}/main"), + false, ) .await { @@ -314,6 +317,7 @@ func Run(req Req) (interface{{}}, error){{ job.timeout, false, &mut Some(occupation_metrics), + None, ) .await?; @@ -378,6 +382,7 @@ pub async fn install_go_dependencies( None, false, &mut Some(occupation_metrics), + None, ) .await?; @@ -444,6 +449,7 @@ pub async fn install_go_dependencies( None, false, &mut Some(occupation_metrics), + None, ) .await?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 7294950363..19c7d53bf7 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -103,6 +103,8 @@ pub async fn handle_child( custom_timeout: Option, sigterm: bool, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // Do not print logs to output, but instead save to string. + pipe_stdout: Option<&mut String>, ) -> error::Result<()> { let start = Instant::now(); @@ -304,6 +306,8 @@ pub async fn handle_child( let mut log_total_size: u64 = 0; let pg_log_total_size = Arc::new(AtomicU32::new(0)); + let mut pipe_stdout = pipe_stdout; + while let Some(line) = output.by_ref().next().await { let do_write_ = do_write.shared(); @@ -385,9 +389,12 @@ pub async fn handle_child( let worker_name = worker.to_string(); let w_id2 = w_id.to_string(); - (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); - - + if let Some(buf) = &mut pipe_stdout { + buf.push_str(&joined); + (do_write, write_result) = tokio::spawn(async { }).remote_handle(); + } else { + (do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle(); + } if let Err(err) = result { tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}"); diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs new file mode 100644 index 0000000000..6f593561d2 --- /dev/null +++ b/backend/windmill-worker/src/java_executor.rs @@ -0,0 +1,915 @@ +use std::{collections::HashMap, path::PathBuf, process::Stdio, sync::Arc}; + +use anyhow::{anyhow, bail}; +use async_recursion::async_recursion; +use itertools::Itertools; +use serde_json::value::RawValue; +use tokio::{ + fs::{create_dir_all, metadata, remove_dir_all, File}, + io::AsyncWriteExt, + process::Command, +}; +use uuid::Uuid; +use windmill_common::{ + error::{self, Error}, + utils::calculate_hash, + worker::{copy_dir_recursively, save_cache, write_file}, +}; +use windmill_parser::Arg; +use windmill_parser_java::parse_java_sig_meta; +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; + +use crate::{ + common::{ + create_args_and_out_file, get_reserved_variables, par_install_language_dependencies, + read_result, start_child_process, OccupancyMetrics, RequiredDependency, + }, + handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, + JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, +}; +lazy_static::lazy_static! { + static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); + static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string()); + static ref JAVAC_PATH: String = std::env::var("JAVAC_PATH").unwrap_or_else(|_| "/usr/bin/javac".to_string()); + static ref CS_PATH: String = std::env::var("COURSIER_PATH").unwrap_or_else(|_| "/usr/bin/coursier".to_string()); + static ref STOREPASS: String = std::env::var("JAVA_STOREPASS").unwrap_or("123456".into()); + static ref TRUST_STORE_PATH: String = std::env::var("JAVA_TRUST_STORE_PATH").unwrap_or("/usr/local/share/ca-certificates/truststore.jks".into()); +} + +const NSJAIL_CONFIG_RUN_JAVA_CONTENT: &str = include_str!("../nsjail/run.java.config.proto"); + +#[allow(dead_code)] +pub(crate) struct JobHandlerInput<'a> { + pub base_internal_url: &'a str, + pub canceled_by: &'a mut Option, + pub client: &'a AuthedClient, + pub parent_runnable_path: Option, + pub db: &'a sqlx::Pool, + pub envs: HashMap, + pub inner_content: &'a str, + pub job: &'a MiniPulledJob, + pub job_dir: &'a str, + pub mem_peak: &'a mut i32, + pub occupancy_metrics: &'a mut OccupancyMetrics, + pub requirements_o: Option<&'a String>, + pub shared_mount: &'a str, + pub worker_name: &'a str, +} + +pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result, Error> { + // --- Prepare --- + { + prepare(&mut args).await?; + } + // --- Generate Lockfile --- + + let deps = resolve( + &args.job.id, + &args.inner_content, + &args.job_dir, + &args.db, + &args.job.workspace_id, + ) + .await?; + + // --- Install --- + + let classpath = install(&mut args, deps).await?; + + // --- Build .java files --- + { + compile(&mut args, &classpath).await?; + } + // --- Run --- + { + run(&mut args, &classpath).await?; + } + // --- Retrieve results --- + { + read_result(&args.job_dir).await + } +} + +async fn prepare<'a>( + JobHandlerInput { job, db, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>, +) -> Result<(), Error> { + // Create needed files + { + create_args_and_out_file(&client, job, job_dir, db).await?; + let app_path = format!("{}/src/main/java/net/script/", job_dir); + create_dir_all(&app_path).await?; + File::create(format!("{app_path}/App.java")) + .await? + .write_all(&wrap(inner_content)?.into_bytes()) + .await?; + File::create(format!("{app_path}/Main.java")) + .await? + .write_all( + &format!( + "package net.script;\n{MINI_CLIENT_IMPORTS}\n{}\n{MINI_CLIENT}", + inner_content + ) + .into_bytes(), + ) + .await?; + } + Ok(()) +} + +pub async fn resolve<'a>( + job_id: &Uuid, + code: &str, + job_dir: &str, + db: &sqlx::Pool, + w_id: &str, +) -> Result { + let deps = { + let find_requirements = code.lines().find_position(|x| { + x.starts_with("//requirements:") || x.starts_with("// requirements:") + }); + + let specified_deps = if let Some((pos, _)) = find_requirements { + code.lines() + .skip(pos + 1) + .map_while(|x| { + if x.starts_with("//") { + Some(x.replace("//", "").trim().to_owned()) + } else { + None + } + }) + .collect::>() + } else { + Default::default() + }; + + let mut deps = vec![ + // Default requirements + "com.fasterxml.jackson.core:jackson-databind:2.9.8".to_owned(), + ]; + deps.extend(specified_deps); + deps.join("\n") + }; + + let req_hash = format!("java-{}", calculate_hash(&deps)); + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + return Ok(cached); + } + let lock = { + append_logs( + job_id, + w_id, + format!("\n--- RESOLVING LOCKFILE ---\n"), + db.clone(), + ) + .await; + + let mut cmd = Command::new(if cfg!(windows) { + "java" + } else { + JAVA_PATH.as_str() + }); + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .env("PATH", PATH_ENV.as_str()) + .envs(PROXY_ENVS.clone()); + + // Configure proxies + { + let jps = parse_proxy()?; + if let Some(val) = jps.https_host { + cmd.arg(&format!("-Dhttps.proxyHost={}", val)); + } + if let Some(val) = jps.https_port { + cmd.arg(&format!("-Dhttps.proxyPort={}", val)); + } + if let Some(val) = jps.http_host { + cmd.arg(&format!("-Dhttp.proxyHost={}", val)); + } + if let Some(val) = jps.http_port { + cmd.arg(&format!("-Dhttp.proxyPort={}", val)); + } + if let Some(val) = jps.no_proxy { + cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); + } + } + if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { + cmd.args(&[ + &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), + &format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS), + ]); + } + cmd.args(&[ + "-jar", + &CS_PATH, + "resolve", + &get_no_default(), + "--parallel", + &format!("{}", *JAVA_CONCURRENT_DOWNLOADS), + "--cache", + COURSIER_CACHE_DIR, + ]) + .args(&get_repos().await) + .args(&deps.split("\n").collect_vec()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + let output = cmd.output().await?; + // Check if the command was successful + if output.status.success() { + String::from_utf8(output.stdout).expect("Failed to convert output to String") + } else { + let stderr = + String::from_utf8(output.stderr).expect("Failed to convert error output to String"); + return Err(error::Error::internal_err(stderr)); + } + }; + + sqlx::query!( + "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", + req_hash, + lock.clone(), + ).fetch_optional(db).await?; + + append_logs(job_id, w_id, format!("\n{}", &lock), db.clone()).await; + Ok(lock) +} + +async fn install<'a>( + JobHandlerInput { worker_name, job, db, job_dir, .. }: &mut JobHandlerInput<'a>, + deps: String, +) -> Result { + let deps = deps + .lines() + .map(|line| { + let unparsed_dep = line.replace(":jar", "").replace(":lib", ""); + let mut it = unparsed_dep.split(":"); + + match (it.next(), it.next(), it.next()) { + (Some(group_id), Some(artifact_id), Some(version)) => { + let path = format!( + "{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}", + group_id.replace(".", "/") + ); + Ok(RequiredDependency { + path, + custom_name: Some(format!("{group_id}:{artifact_id}:{version}")), + short_name: Some(format!("{artifact_id}:{version}")), + }) + } + _ => anyhow::bail!("{line} is not parsable"), + } + }) + .collect::>>()?; + + let classpath = deps + .clone() + .into_iter() + .map(|RequiredDependency { path, .. }| path + "/*") + .collect_vec() + .join(":") + + ":target"; + + #[cfg(windows)] + let classpath = classpath.replace(":", ";"); + + tracing::debug!( + workspace_id = %job.workspace_id, + "JAVA classpath: {}", &classpath + ); + let (repos, no_default, trust_store_metadata) = ( + get_repos().await, + get_no_default(), + metadata(TRUST_STORE_PATH.clone()).await, + ); + let job_dir = job_dir.to_owned(); + let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4()); + let fetch_dir2 = fetch_dir.clone(); + par_install_language_dependencies( + deps, + "java", + "java", + true, + *JAVA_CONCURRENT_DOWNLOADS, + true, + crate::common::InstallStrategy::AllAtOnce(Arc::new(move |dependencies| { + let mut cmd = Command::new(if cfg!(windows) { + "java" + } else { + JAVA_PATH.as_str() + }); + let artifacts = dependencies + .into_iter() + .map(|e| { + e.custom_name.ok_or(anyhow::anyhow!( + "Internal Error: Artifact name should be Some!" + )) + }) + .collect::>>()?; + cmd.env_clear() + .current_dir(&job_dir) + .env("PATH", PATH_ENV.as_str()) + .envs(PROXY_ENVS.clone()); + // Configure proxies + { + let jps = parse_proxy()?; + if let Some(val) = jps.https_host { + cmd.arg(&format!("-Dhttps.proxyHost={}", val)); + } + if let Some(val) = jps.https_port { + cmd.arg(&format!("-Dhttps.proxyPort={}", val)); + } + if let Some(val) = jps.http_host { + cmd.arg(&format!("-Dhttp.proxyHost={}", val)); + } + if let Some(val) = jps.http_port { + cmd.arg(&format!("-Dhttp.proxyPort={}", val)); + } + if let Some(val) = jps.no_proxy { + cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); + } + } + + if trust_store_metadata.is_ok() { + cmd.args(&[ + &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), + &format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS), + ]); + } + cmd.args(&[ + "-jar", + &CS_PATH, + "fetch", + &no_default, + "--quiet", + "--parallel", + &format!("{}", *JAVA_CONCURRENT_DOWNLOADS), + "--cache", + &fetch_dir, + ]) + .args(&repos) + .arg("--intransitive") + .args(artifacts) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + + Ok(cmd) + })), + async move |_| { + move_to_repository(&fetch_dir2, 0).await?; + remove_dir_all(&fetch_dir2).await?; + #[async_recursion] + async fn move_to_repository(path: &str, depth: u8) -> anyhow::Result<()> { + if depth == 3 { + copy_dir_recursively( + &PathBuf::from(path), + &PathBuf::from(JAVA_REPOSITORY_DIR), + )?; + + return Ok(()); + } + let mut entries = tokio::fs::read_dir(path).await?; + loop { + let Some(entry) = entries.next_entry().await? else { + break Ok(()); + }; + + let path = entry + .path() + .to_str() + .ok_or(anyhow!("Internal Error: Cannot convert Path to Str"))? + .to_owned(); + + move_to_repository(&path, depth + 1).await?; + } + } + Ok(()) + }, + &job.id, + &job.workspace_id, + worker_name, + db, + ) + .await?; + Ok(classpath) +} + +async fn compile<'a>( + JobHandlerInput { + occupancy_metrics, + mem_peak, + canceled_by, + worker_name, + job, + db, + job_dir, + client, + envs, + base_internal_url, + inner_content, + requirements_o, + parent_runnable_path, + .. + }: &mut JobHandlerInput<'a>, + classpath: &'a str, + // plugins: Vec<&'a str>, +) -> Result<(), Error> { + fn compute_hash(code: &str, requirements_o: Option<&String>) -> String { + calculate_hash(&format!( + "{}{}", + code, + requirements_o + .as_ref() + .map(|x| x.to_string()) + .unwrap_or_default() + )) + } + let reserved_variables = + get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?; + let hash = compute_hash(inner_content, *requirements_o); + let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR); + let remote_path = format!("java_jar/{hash}"); + let (cache, ..) = windmill_common::worker::load_cache(&bin_path, &remote_path, true).await; + + if cache { + let target = format!("{job_dir}/target"); + + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = copy_dir_recursively(&PathBuf::from(&bin_path), &PathBuf::from(&target)); + + symlink.map_err(|e| { + Error::ExecutionErr(format!( + "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + )) + })?; + } else { + // let plugin_registry = format!("{job_dir}/plugin-registry"); + let child = { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- COMPILING .JAVA FILES\n"), + db.clone(), + ) + .await; + + let mut cmd = Command::new(if cfg!(windows) { + "javac" + } else { + JAVAC_PATH.as_str() + }); + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(envs) + .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) + .args(&[ + "-classpath", + &classpath, + "src/main/java/net/script/Main.java", + "src/main/java/net/script/App.java", + "-d", + "./target", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + start_child_process(cmd, "javac").await? + }; + handle_child::handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "javac", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + ) + .await?; + + match save_cache( + &bin_path, + &format!("java_jar/{hash}"), + &format!("{job_dir}/target"), + true, + ) + .await + { + Err(e) => { + let em = format!( + "could not save {bin_path} to {} to java cache: {e:?}", + format!("{job_dir}/main"), + ); + tracing::error!(em); + } + Ok(logs) => { + tracing::trace!(logs); + } + } + }; + + Ok(()) +} +async fn run<'a>( + JobHandlerInput { + occupancy_metrics, + mem_peak, + canceled_by, + worker_name, + job, + db, + job_dir, + shared_mount, + client, + envs, + base_internal_url, + parent_runnable_path, + .. + }: &mut JobHandlerInput<'a>, + classpath: &'a str, +) -> Result<(), Error> { + let reserved_variables = + get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?; + + let child = if !cfg!(windows) && !*DISABLE_NSJAIL { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- ISOLATED JAVA CODE EXECUTION ---\n"), + db.clone(), + ) + .await; + + write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_JAVA_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", JAVA_CACHE_DIR) + .replace("{SHARED_MOUNT}", &shared_mount) + // .replace("{CACHED_TARGET}", &shared_mount) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), + )?; + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.env_clear() + .current_dir(job_dir) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(envs) + .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) + .args(vec![ + "--config", + "run.config.proto", + "--", + JAVA_PATH.as_str(), + ]); + if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { + cmd.args(&[ + &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), + &format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS), + ]); + } + // Configure proxies + { + let jps = parse_proxy()?; + if let Some(val) = jps.https_host { + cmd.arg(&format!("-Dhttps.proxyHost={}", val)); + } + if let Some(val) = jps.https_port { + cmd.arg(&format!("-Dhttps.proxyPort={}", val)); + } + if let Some(val) = jps.http_host { + cmd.arg(&format!("-Dhttp.proxyHost={}", val)); + } + if let Some(val) = jps.http_port { + cmd.arg(&format!("-Dhttp.proxyPort={}", val)); + } + if let Some(val) = jps.no_proxy { + cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); + } + } + cmd.args(vec!["-classpath", &classpath, "net.script.App"]); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + + start_child_process(cmd, NSJAIL_PATH.as_str()).await? + } else { + append_logs( + &job.id, + &job.workspace_id, + format!("\n--- JAVA CODE EXECUTION ---\n"), + db.clone(), + ) + .await; + + let mut cmd = Command::new(if cfg!(windows) { + "java" + } else { + JAVA_PATH.as_str() + }); + cmd.env_clear() + .current_dir(job_dir.to_owned()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(envs) + .envs(reserved_variables); + if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { + cmd.args(&[ + &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), + &format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS), + ]); + } + // Configure proxies + { + let jps = parse_proxy()?; + if let Some(val) = jps.https_host { + cmd.arg(&format!("-Dhttps.proxyHost={}", val)); + } + if let Some(val) = jps.https_port { + cmd.arg(&format!("-Dhttps.proxyPort={}", val)); + } + if let Some(val) = jps.http_host { + cmd.arg(&format!("-Dhttp.proxyHost={}", val)); + } + if let Some(val) = jps.http_port { + cmd.arg(&format!("-Dhttp.proxyPort={}", val)); + } + if let Some(val) = jps.no_proxy { + cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val)); + } + } + cmd.args(&["-classpath", &classpath, "net.script.App"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ); + } + start_child_process(cmd, "java").await? + }; + handle_child::handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "java", + job.timeout, + false, + &mut Some(occupancy_metrics), + None, + ) + .await +} + +#[derive(Default, Debug)] +struct JavaProxySettings { + http_host: Option, + http_port: Option, + https_host: Option, + https_port: Option, + no_proxy: Option, +} +fn parse_proxy() -> anyhow::Result { + let mut jps = JavaProxySettings::default(); + for (ident, mut val) in PROXY_ENVS.clone() { + match ident { + "HTTPS_PROXY" => { + if val.contains("http://") { + bail!("HTTPS_PROXY url cannot contain http scheme."); + } + if !val.contains("https://") { + val = format!("https://{val}"); + } + let mut url = url::Url::parse(&val)?; + let port = url.port(); + // Make sure port and schema is not included in final url + { + url.set_port(None).unwrap_or_default(); + jps.https_host = Some(url.as_str().replace("https://", "")); + if let Some(port) = port { + jps.https_port = Some(format!("{}", port)); + } + } + } + "HTTP_PROXY" => { + if val.contains("https://") { + bail!("HTTP_PROXY url cannot contain https scheme."); + } + if !val.contains("http://") { + val = format!("http://{val}"); + } + let mut url = url::Url::parse(&val)?; + let port = url.port(); + // Make sure port and schema is not included in final url + { + url.set_port(None).unwrap_or_default(); + jps.http_host = Some(url.as_str().replace("http://", "")); + if let Some(port) = port { + jps.https_port = Some(format!("{}", port)); + } + } + } + // Java uses | instead of , + "NO_PROXY" => jps.no_proxy = Some(val.replace(",", "|")), + _ => {} + } + } + + Ok(jps) +} +async fn get_repos() -> Vec { + MAVEN_REPOS + .read() + .await + .as_ref() + .map(|repos| { + repos + .trim() + .split_whitespace() + .into_iter() + .map(|el| vec!["--repository".to_owned(), el.to_owned()]) + .collect_vec() + }) + .unwrap_or_default() + .concat() +} + +fn get_no_default() -> String { + if NO_DEFAULT_MAVEN.load(std::sync::atomic::Ordering::Relaxed) { + "--no-default" + } else { + // Command does not take empty arguments + "-q" + } + .into() +} + +/// Wraps content script +/// that upon execution reads args.json (which are piped and transformed from previous flow step or top level inputs) +/// Also wrapper takes output of program and serializes to result.json (Which windmill will know how to use later) +fn wrap(inner_content: &str) -> Result { + let sig = parse_java_sig_meta(inner_content)?; + let ret_void = sig.returns_void; + let spread = sig + .main_sig + .args + .clone() + .into_iter() + .map(|Arg { name, .. }| { + // Apply additional input transformation + format!(" parsedArgs.{name}") + }) + .collect_vec() + .join(","); + let args = sig + .main_sig + .args + .clone() + .into_iter() + .map(|Arg { name, otyp, .. }| { + // Apply additional input transformation + format!("public {} {name};\n", otyp.unwrap()) + }) + .collect_vec() + .join(" "); + Ok(r#" +package net.script; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.FileOutputStream; +import net.script.Main; + +public class App{ + + public static class Args {ARGS} + + public static void main(String[] args) { + try { + InputStream fileInputStream = new FileInputStream("args.json"); + ObjectMapper mapper = new ObjectMapper(); + Args parsedArgs = mapper.readValue(fileInputStream, Args.class); + fileInputStream.close(); + {MAIN_HANDLER} + FileOutputStream fileOutputStream = new FileOutputStream("result.json"); + mapper.writeValue(fileOutputStream, res); + fileOutputStream.close(); + + } catch (Exception e) { // Catching general Exception + e.printStackTrace(); // Handle the exception + } + } +} + "# + .replace( + "{MAIN_HANDLER}", + if ret_void { + " + Main.main(SPREAD); + Object res = null; + " + } else { + " + Object res = Main.main(SPREAD); + " + }, + ) + .replace("SPREAD", &spread) + .replace("ARGS", &args)) +} +const MINI_CLIENT_IMPORTS: &str = r#" +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +"#; +const MINI_CLIENT: &str = r#" +class Wmill { + public static String getVariable(String path) { + var baseUrl = System.getenv("BASE_INTERNAL_URL"); + var workspace = System.getenv("WM_WORKSPACE"); + var uri = java.text.MessageFormat.format("{0}/api/w/{1}/variables/get_value/{2}", baseUrl, workspace, path); + + // Create an HttpRequest + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(uri)) + .header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header + .GET() // Set the request method to GET + .build(); + + // Send the request and get the response + return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .thenApply(HttpResponse::body) + .join(); // Wait for the completion + } + public static String getResource(String path) { + var baseUrl = System.getenv("BASE_INTERNAL_URL"); + var workspace = System.getenv("WM_WORKSPACE"); + var uri = java.text.MessageFormat.format("{0}/api/w/{1}/resources/get_value_interpolated/{2}", baseUrl, workspace, path); + + // Create an HttpRequest + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(uri)) + .header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header + .GET() // Set the request method to GET + .build(); + + // Send the request and get the response + return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .thenApply(HttpResponse::body) + .join(); // Wait for the completion + } +} +"#; diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index ba47fd81fc..607616117f 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -11,6 +11,10 @@ mod bash_executor; #[cfg(feature = "nu")] mod nu_executor; + +#[cfg(feature = "java")] +mod java_executor; + #[cfg(feature = "benchmark")] pub mod bench; mod bun_executor; @@ -40,11 +44,11 @@ mod python_executor; mod result_processor; #[cfg(feature = "rust")] mod rust_executor; +mod sanitized_sql_params; +mod schema; mod worker; mod worker_flow; mod worker_lockfiles; -mod sanitized_sql_params; -mod schema; pub use worker::*; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index e40e1029f4..729302b7c3 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -334,6 +334,7 @@ async fn run<'a>( job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await } diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 19dce7f1b9..4707c8db16 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -105,6 +105,7 @@ pub async fn composer_install( None, false, &mut Some(occupancy_metrics), + None, ) .await?; @@ -339,6 +340,7 @@ try {{ job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; read_result(job_dir).await diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 648c40a9a3..e3bfe6838b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -27,7 +27,7 @@ use windmill_common::{ Error::{self}, }, utils::calculate_hash, - worker::{write_file, PythonAnnotations, WORKER_CONFIG}, + worker::{copy_dir_recursively, pad_string, write_file, PythonAnnotations, WORKER_CONFIG}, DB, }; @@ -49,6 +49,7 @@ lazy_static::lazy_static! { static ref PY_CONCURRENT_DOWNLOADS: usize = var("PY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); + static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap(); static ref TRUSTED_HOST: Option = var("PY_TRUSTED_HOST").ok().or(var("PIP_TRUSTED_HOST").ok()); @@ -350,6 +351,7 @@ impl PyVersion { None, false, occupancy_metrics, + None, ) .await } @@ -644,6 +646,7 @@ pub async fn uv_pip_compile( None, false, occupancy_metrics, + None, ) .await .map_err(|e| { @@ -780,30 +783,6 @@ async fn postinstall( Ok(()) } -fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Result<()> { - if !dst.exists() { - fs::create_dir_all(dst)?; - } - - tracing::debug!("Copying recursively from {:?} to {:?}", src, dst); - - for entry in fs::read_dir(src)? { - let entry = entry?; - let src_path = entry.path(); - let dst_path = dst.join(entry.file_name()); - - if src_path.is_dir() && !src_path.is_symlink() { - copy_dir_recursively(&src_path, &dst_path)?; - } else { - fs::copy(&src_path, &dst_path)?; - } - } - - tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst); - - Ok(()) -} - async fn get_python_path( py_version: PyVersion, worker_name: &str, @@ -1161,6 +1140,7 @@ mount {{ job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; @@ -1709,19 +1689,6 @@ async fn spawn_uv_install( } } -/// length = 5 -/// value = "foo" -/// output = "foo " -/// 12345 -fn pad_string(value: &str, total_length: usize) -> String { - if value.len() >= total_length { - value.to_string() // Return the original string if it's already long enough - } else { - let padding_needed = total_length - value.len(); - format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces - } -} - /// uv pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, @@ -2053,7 +2020,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level()) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -2204,7 +2171,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level())); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false)); } } diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index d2a0c3217d..21ec46422d 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -164,6 +164,7 @@ pub async fn generate_cargo_lockfile( None, false, &mut Some(occupancy_metrics), + None, ) .await?; @@ -226,6 +227,7 @@ pub async fn build_rust_crate( None, false, &mut Some(occupancy_metrics), + None, ) .await?; append_logs(job_id, w_id, "\n\n", db).await; @@ -245,6 +247,7 @@ pub async fn build_rust_crate( &bin_path, &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), &format!("{job_dir}/main"), + false, ) .await { @@ -294,7 +297,8 @@ pub async fn handle_rust_job( let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); - let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await; + let (cache, cache_logs) = + windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { let target = format!("{job_dir}/main"); @@ -407,6 +411,7 @@ pub async fn handle_rust_job( job.timeout, false, &mut Some(occupancy_metrics), + None, ) .await?; read_result(job_dir).await diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a1f1754ab5..f13d22fafe 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -123,7 +123,10 @@ use crate::{ use crate::rust_executor::handle_rust_job; #[cfg(feature = "nu")] -use crate::nu_executor::{handle_nu_job, JobHandlerInput}; +use crate::nu_executor::{handle_nu_job, JobHandlerInput as JobHandlerInputNu}; + +#[cfg(feature = "java")] +use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJava}; #[cfg(feature = "php")] use crate::php_executor::handle_php_job; @@ -264,6 +267,8 @@ pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311" pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); +pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); + pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); @@ -275,6 +280,12 @@ pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu"); pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); + +// JAVA +pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java"); +pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache"); +pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository"); +// KJQXZ pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle"); @@ -398,6 +409,11 @@ lazy_static::lazy_static! { pub static ref NPM_CONFIG_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); pub static ref BUNFIG_INSTALL_SCOPES: Arc>> = Arc::new(RwLock::new(None)); pub static ref NUGET_CONFIG: Arc>> = Arc::new(RwLock::new(None)); + pub static ref MAVEN_REPOS: Arc>> = Arc::new(RwLock::new(None)); + pub static ref NO_DEFAULT_MAVEN: AtomicBool = AtomicBool::new(std::env::var("NO_DEFAULT_MAVEN") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false)); pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); @@ -3078,7 +3094,33 @@ mount {{ ); #[cfg(feature = "nu")] - handle_nu_job(JobHandlerInput { + handle_nu_job(JobHandlerInputNu { + mem_peak, + canceled_by, + job, + db, + client, + parent_runnable_path, + inner_content: &code, + job_dir, + requirements_o: lock.as_ref(), + shared_mount: &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + }) + .await + } + Some(ScriptLang::Java) => { + #[cfg(not(feature = "java"))] + return Err(anyhow::anyhow!( + "Java is not available because the feature is not enabled" + ) + .into()); + + #[cfg(feature = "java")] + handle_java_job(JobHandlerInputJava { mem_peak, canceled_by, job, @@ -3163,6 +3205,11 @@ fn parse_sig_of_lang( ScriptLang::Nu => Some(windmill_parser_nu::parse_nu_signature(code)?), #[cfg(not(feature = "nu"))] ScriptLang::Nu => None, + #[cfg(feature = "java")] + ScriptLang::Java => Some(windmill_parser_java::parse_java_signature(code)?), + #[cfg(not(feature = "java"))] + ScriptLang::Java => None, + // KJQXZ } } else { None diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 17ac95ab3e..3b2d04c759 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -36,6 +36,9 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel} use crate::common::OccupancyMetrics; use crate::csharp_executor::generate_nuget_lockfile; +#[cfg(feature = "java")] +use crate::java_executor::resolve; + #[cfg(feature = "php")] use crate::php_executor::{composer_install, parse_php_imports}; #[cfg(feature = "python")] @@ -2076,16 +2079,17 @@ async fn capture_dependency_job( ) .await } - ScriptLang::Postgresql => Ok("".to_owned()), - ScriptLang::Mysql => Ok("".to_owned()), - ScriptLang::Bigquery => Ok("".to_owned()), - ScriptLang::Snowflake => Ok("".to_owned()), - ScriptLang::Mssql => Ok("".to_owned()), - ScriptLang::Graphql => Ok("".to_owned()), - ScriptLang::OracleDB => Ok("".to_owned()), - ScriptLang::Bash => Ok("".to_owned()), - ScriptLang::Nu => Ok("".to_owned()), - ScriptLang::Powershell => Ok("".to_owned()), - ScriptLang::Nativets => Ok("".to_owned()), + #[cfg(feature = "java")] + ScriptLang::Java => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for Java".to_string(), + )); + } + + resolve(job_id, job_raw_code, job_dir, db, w_id).await + } + // KJQXZ + _ => Ok("".to_owned()), } } diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 5c2247e66b..d711496ba7 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -119,4 +119,12 @@ inventory: debug: msg: "Hello, world!" `, + java: ` +public class Main { + public static void main() { + System.out.println("Hello World"); + } +} +`, +// KJQXZ }; diff --git a/cli/dnt.ts b/cli/dnt.ts index 64d9957499..3820439a7e 100644 --- a/cli/dnt.ts +++ b/cli/dnt.ts @@ -69,6 +69,7 @@ await build({ "rust", "yaml", "csharp", + "java", ]; for (const l of dirs) { diff --git a/cli/gen/services.gen.ts b/cli/gen/services.gen.ts index bee9069a5b..6771928fd7 100644 --- a/cli/gen/services.gen.ts +++ b/cli/gen/services.gen.ts @@ -8652,4 +8652,4 @@ export const clearIndex = (data: ClearIndexData): CancelablePromise { throw Error('TextEncoder not available') } } ); + +const encodeString = function (arg, view) { + return cachedTextEncoder.encodeInto(arg, view); +}; + +function passStringToWasm0(arg, malloc, realloc) { + + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = encodeString(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); + +if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} +/** + * @param {string} code + * @returns {string} + */ +export function parse_java(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_java(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +const imports = { + __wbindgen_placeholder__: { + __wbindgen_init_externref_table: function() { + const table = wasm.__wbindgen_export_0; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + ; + }, + }, + +}; + +const wasm_url = new URL('windmill_parser_wasm_bg.wasm', import.meta.url); +let wasmCode = ''; +switch (wasm_url.protocol) { + case 'file:': + wasmCode = await Deno.readFile(wasm_url); + break + case 'https:': + case 'http:': + wasmCode = await (await fetch(wasm_url)).arrayBuffer(); + break + default: + throw new Error(`Unsupported protocol: ${wasm_url.protocol}`); +} + +const wasmInstance = (await WebAssembly.instantiate(wasmCode, imports)).instance; +const wasm = wasmInstance.exports; +export const __wasm = wasm; + +wasm.__wbindgen_start(); + diff --git a/cli/wasm/java/windmill_parser_wasm_bg.wasm b/cli/wasm/java/windmill_parser_wasm_bg.wasm new file mode 100644 index 0000000000..5395c1e789 Binary files /dev/null and b/cli/wasm/java/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/java/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/java/windmill_parser_wasm_bg.wasm.d.ts new file mode 100644 index 0000000000..6128cc38c6 --- /dev/null +++ b/cli/wasm/java/windmill_parser_wasm_bg.wasm.d.ts @@ -0,0 +1,29 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const parse_java: (a: number, b: number) => [number, number]; +export const abort: () => void; +export const malloc: (a: number) => number; +export const calloc: (a: number, b: number) => number; +export const realloc: (a: number, b: number) => number; +export const free: (a: number) => void; +export const strncmp: (a: number, b: number, c: number) => number; +export const iswspace: (a: number) => number; +export const iswalnum: (a: number) => number; +export const clock: () => bigint; +export const isprint: (a: number) => number; +export const fprintf: (a: number, b: number, c: number) => number; +export const fputs: (a: number, b: number) => number; +export const fputc: (a: number, b: number) => number; +export const fdopen: (a: number, b: number) => number; +export const fclose: (a: number) => number; +export const fwrite: (a: number, b: number, c: number, d: number) => number; +export const vsnprintf: (a: number, b: number, c: number, d: number) => number; +export const clock_gettime: (a: number, b: number) => void; +export const snprintf: () => void; +export const __assert_fail: (a: number, b: number, c: number, d: number) => void; +export const __wbindgen_export_0: WebAssembly.Table; +export const __wbindgen_malloc: (a: number, b: number) => number; +export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_free: (a: number, b: number, c: number) => void; +export const __wbindgen_start: () => void; diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 7f904b01b7..c36682c4e9 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -14,3 +14,10 @@ ENV DOTNET_ROOT="/opt/dotnet-sdk/bin" # Nushell COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu + +# Java +RUN apt-get -y update && apt-get install -y default-jdk +RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \ + && mv ./coursier /usr/bin/coursier \ + && chmod +x /usr/bin/coursier +RUN /usr/bin/java -jar /usr/bin/coursier about diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index b074bb4ecf..25cc47b2dd 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -37,3 +37,10 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc # Nushell COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu + +# Java +RUN apt-get -y update && apt-get install -y default-jdk +RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \ + && mv ./coursier /usr/bin/coursier \ + && chmod +x /usr/bin/coursier +RUN /usr/bin/java -jar /usr/bin/coursier about diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index d667b15722..e51e73a3ea 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -62,4 +62,12 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc # Nushell COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu +# Java +RUN apt-get install -y default-jdk +RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \ + && mv ./coursier /usr/bin/coursier \ + && chmod +x /usr/bin/coursier +RUN /usr/bin/java -jar /usr/bin/coursier about + COPY --from=nsjail /nsjail/nsjail /bin/nsjail + diff --git a/flake.nix b/flake.nix index 506e61d68e..8ded5a3ac9 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,13 @@ pkg-config cmake ]; + coursier = pkgs.fetchFromGitHub { + owner = "coursier"; + repo = "launchers"; + rev = "79d927f7586c09ca6d8cd01862adb0d9f9d88dff"; + hash = "sha256-8E0WtDFc7RcqmftDigMyy1xXUkjgL4X4kpf7h1GdE48="; + }; + PKG_CONFIG_PATH = pkgs.lib.makeSearchPath "lib/pkgconfig" (with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev ]); RUSTY_V8_ARCHIVE = let @@ -75,6 +82,7 @@ sqlx-cli flock sccache + nsjail deno python3 python3Packages.pip @@ -84,7 +92,9 @@ nushell dotnet-sdk_9 oracle-instantclient + # LSP/Local dev svelte-language-server + taplo ]); packages = [ (pkgs.writeScriptBin "wm-caddy" '' @@ -126,6 +136,25 @@ cd ./frontend npm run dev $* '') + (pkgs.writeScriptBin "wm-minio" '' + set -e + cd ./backend + mkdir -p .minio-data/wmill + ${pkgs.minio}/bin/minio server ./.minio-data + '') + # Generate keys + # TODO: Do not set new keys if ran multiple times + (pkgs.writeScriptBin "wm-minio-keys" '' + set -e + cd ./backend + ${pkgs.minio-client}/bin/mc alias set 'wmill-minio-dev' 'http://localhost:9000' 'minioadmin' 'minioadmin' + ${pkgs.minio-client}/bin/mc admin accesskey create myminio | tee .minio-data/secrets.txt + echo "" + echo 'Saving to: ./backend/.minio-data/secrets.txt' + echo "bucket: wmill" + echo "endpoint: http://localhost:9000" + '') + ]; inherit PKG_CONFIG_PATH RUSTY_V8_ARCHIVE; @@ -140,11 +169,16 @@ BUN_PATH = "${pkgs.bun}/bin/bun"; UV_PATH = "${pkgs.uv}/bin/uv"; NU_PATH = "${pkgs.nushell}/bin/nu"; + JAVA_PATH = "${pkgs.jdk21}/bin/java"; + JAVAC_PATH = "${pkgs.jdk21}/bin/javac"; + COURSIER_PATH = "${coursier}/coursier"; + # KJQXZ FLOCK_PATH = "${pkgs.flock}/bin/flock"; CARGO_PATH = "${rust}/bin/cargo"; DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet"; DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet"; ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib"; + RUST_LOG = "debug"; }; packages.default = self.packages.${system}.windmill; packages.windmill-client = pkgs.buildNpmPackage { @@ -218,7 +252,6 @@ "cloud" "jemalloc" "tantivy" - "deno_core" "license" "http_trigger" "zip" @@ -226,16 +259,10 @@ "kafka" "otel" "dind" - "php" - "mysql" - "mssql" - "bigquery" "websocket" - "python" "smtp" - "csharp" "static_frontend" - "rust" + "all_languages" ]; doCheck = false; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b31aeec507..613597544c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -68,6 +68,7 @@ "vscode-ws-jsonrpc": "~3.4.0", "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", + "windmill-parser-wasm-java": "^1.478.1", "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", @@ -12528,6 +12529,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.429.0.tgz", "integrity": "sha512-M3jeGDqeTyPj9HyyX3msdzMrqIIzlfMfxTMsXS8m7MJp4Cm60qifMxD29Ipxb2B4WdzyGwCSlaBjLsXu0b3c5g==" }, + "node_modules/windmill-parser-wasm-java": { + "version": "1.478.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-java/-/windmill-parser-wasm-java-1.478.1.tgz", + "integrity": "sha512-2gMr5pXaEExoLjeSMOpHzY4UjDU/JGjvlXSnBiQDHUaEoKSS7FWa5APzBKcmn8cM+mr2p8m4sMCyTXS88At0fQ==" + }, "node_modules/windmill-parser-wasm-nu": { "version": "1.474.1", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-nu/-/windmill-parser-wasm-nu-1.474.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 489dba9869..d404efee87 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -144,6 +144,7 @@ "vscode-ws-jsonrpc": "~3.4.0", "windmill-parser-wasm-csharp": "^1.437.1", "windmill-parser-wasm-go": "^1.429.0", + "windmill-parser-wasm-java": "^1.478.1", "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 807e16e69b..5b2aff62ed 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -195,6 +195,8 @@ | 'yaml' | 'csharp' | 'nu' + | 'java' + // KJQXZ export let code: string = '' export let cmdEnterAction: (() => void) | undefined = undefined export let formatAction: (() => void) | undefined = undefined diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 3a778bfa66..9f785760af 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -93,7 +93,9 @@ 'php', 'rust', 'csharp', - 'nu' + 'nu', + 'java', + // KJQXZ ].includes(lang ?? '') $: showVarPicker = [ 'python3', @@ -107,7 +109,9 @@ 'php', 'rust', 'csharp', - 'nu' + 'nu', + 'java', + // KJQXZ ].includes(lang ?? '') $: showResourcePicker = [ 'python3', @@ -121,7 +125,9 @@ 'php', 'rust', 'csharp', - 'nu' + 'nu', + 'java', + // KJQXZ ].includes(lang ?? '') $: showResourceTypePicker = ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) || @@ -373,6 +379,9 @@ editor.insertAtCursor(`Environment.GetEnvironmentVariable("${name}");`) } else if (lang == 'nu') { editor.insertAtCursor(`$env.${name}`) + } else if (lang == 'java') { + editor.insertAtCursor(`System.getenv("${name}");`) + // KJQXZ } sendUserToast(`${name} inserted at cursor`) }} @@ -440,6 +449,9 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri); `) } else if (lang == 'nu') { editor.insertAtCursor(`get_variable ${path}`) + } else if (lang == 'java') { + editor.insertAtCursor(`(Wmill.getVariable("${path}"))`) + // KJQXZ } sendUserToast(`${name} inserted at cursor`) }} @@ -524,7 +536,11 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS `) } else if (lang == 'nu') { editor.insertAtCursor(`get_resource ${path}`) + } else if (lang == 'java') { + editor.insertAtCursor(`(Wmill.getResource("${path}"))`) + // KJQXZ } + sendUserToast(`${path} inserted at cursor`) }} tooltip="Resources represent connections to third party systems. Resources are a good way to define a connection to a frequently used third party system such as a database." diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 87daeed695..d7bdc18314 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -12,6 +12,7 @@ import rust from 'svelte-highlight/languages/rust' import csharp from 'svelte-highlight/languages/csharp' import yaml from 'svelte-highlight/languages/yaml' + import java from 'svelte-highlight/languages/java' import type { Script } from '$lib/gen' import { Button } from './common' import { copyToClipboard } from '$lib/utils' @@ -65,7 +66,10 @@ case 'nu': return python case 'ansible': - return yaml + return yaml; + case 'java': + return java; + // KJQXZ default: return typescript } diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index eec97085fe..e68ee95c79 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -78,7 +78,7 @@ deno: false, go: false, ruff: false, - shellcheck: false + shellcheck: false, } const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 55878d4928..f64357e1cc 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -123,7 +123,9 @@ 'rust', 'ansible', 'csharp', - 'nu' + 'nu', + 'java' + // KJQXZ ] const nativeTags = [ 'nativets', diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 1c75ce39ed..38b3e12260 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -21,6 +21,7 @@ import AnsibleIcon from '$lib/components/icons/AnsibleIcon.svelte' import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte' import NuIcon from '$lib/components/icons/NuIcon.svelte' + import JavaIcon from '$lib/components/icons/JavaIcon.svelte' export let lang: | SupportedLanguage @@ -56,7 +57,9 @@ rust: 'Rust', ansible: 'Ansible Playbook', csharp: 'C#', - nu: 'Nu' + nu: 'Nu', + java: 'Java' + // KJQXZ } const langToComponent: Record< @@ -87,7 +90,9 @@ rust: RustIcon, ansible: AnsibleIcon, csharp: CSharpIcon, - nu: NuIcon + nu: NuIcon, + java: JavaIcon + // KJQXZ } let subIconScale = width === 30 ? 0.6 : 0.8 diff --git a/frontend/src/lib/components/icons/JavaIcon.svelte b/frontend/src/lib/components/icons/JavaIcon.svelte new file mode 100644 index 0000000000..868aeeccaa --- /dev/null +++ b/frontend/src/lib/components/icons/JavaIcon.svelte @@ -0,0 +1,6 @@ + + + diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 993e794e7f..f442b976cc 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -302,7 +302,24 @@ export const settings: Record = { codeAreaLang: 'xml', storage: 'setting', ee_only: '' - } + }, + { + label: 'Maven/Ivy repositories', + description: 'Add private Maven/Ivy repositories', + key: 'maven_repos', + fieldType: 'text', + placeholder: 'https://user:password@artifacts.foo.com/maven', + storage: 'setting', + ee_only: '' + }, + { + label: 'Disable default Maven repository', + description: 'Do not use default Maven repository', + key: 'no_default_maven', + fieldType: 'boolean', + storage: 'setting', + ee_only: '' + }, ], Alerts: [ { diff --git a/frontend/src/lib/editorUtils.ts b/frontend/src/lib/editorUtils.ts index 12ebc7f0b1..2b8ddb6ba4 100644 --- a/frontend/src/lib/editorUtils.ts +++ b/frontend/src/lib/editorUtils.ts @@ -84,6 +84,9 @@ export function langToExt(lang: string): string { return 'cs' case 'nu': return 'nu' + case 'java': + return 'java' + // KJQXZ default: return 'unknown' } diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 4c0e55fa94..3fa9c3053e 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -24,6 +24,7 @@ import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust' import initYamlParser, { parse_ansible } from 'windmill-parser-wasm-yaml' import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp' import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu' +import initJavaParser, { parse_java } from 'windmill-parser-wasm-java' import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url' import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url' @@ -34,6 +35,7 @@ import wasmUrlRust from 'windmill-parser-wasm-rust/windmill_parser_wasm_bg.wasm? import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm?url' import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.wasm?url' import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url' +import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url' import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from './inferArgSig.js' @@ -71,6 +73,9 @@ async function initWasmCSharp() { async function initWasmNu() { await initNuParser(wasmUrlNu) } +async function initWasmJava() { + await initJavaParser(wasmUrlJava) +} export async function inferArgs( language: SupportedLanguage | 'bunnative' | undefined, @@ -186,6 +191,10 @@ export async function inferArgs( } else if (language == 'nu') { await initWasmNu() inferedSchema = JSON.parse(parse_nu(code)) + } else if (language == 'java') { + await initWasmJava() + inferedSchema = JSON.parse(parse_java(code)) + // KJQXZ } else { return null } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index bdb5ce0870..1fad760e04 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -945,6 +945,61 @@ dependencies: content: "{{ my_result | to_json }}" dest: result.json ` +const JAVA_INIT_CODE = `//requirements: +//com.google.code.gson:gson:2.8.9 +//com.github.ricksbrown:cowsay:1.1.0 + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.github.ricksbrown.cowsay.Cowsay; +import com.github.ricksbrown.cowsay.plugin.CowExecutor; + +public class Main { + public static class Person { + private String name; + private int age; + + // Constructor + public Person(String name, int age) { + this.name = name; + this.age = age; + } + } + + public static Object main( + // Primitive + int a, + float b, + // Objects + Integer age, + Float d, + Object e, + String name, + // Lists + String[] f + // No trailing commas! + ){ + Gson gson = new Gson(); + + // Get resources + var theme = Wmill.getResource("f/app_themes/theme_0"); + System.out.println("Theme: " + theme); + + // Create a Person object + Person person = new Person( (name == "") ? "Alice" : name, (age == null) ? 30 : age); + + // Serialize the Person object to JSON + String json = gson.toJson(person); + System.out.println("Serialized JSON: " + json); + + // Use cowsay + String[] args = new String[]{"-f", "dragon", json }; + String result = Cowsay.say(args); + return result; + } +} +` +// KJQXZ export const INITIAL_CODE = { bun: { scriptInitCodeBlock: BUN_INIT_BLOCK, @@ -1028,7 +1083,11 @@ export const INITIAL_CODE = { }, bunnative: { script: BUNNATIVE_INIT_CODE - } + }, + java: { + script: JAVA_INIT_CODE + }, + // KJQXZ } export function isInitialCode(content: string): boolean { @@ -1132,6 +1191,9 @@ export function initialCode( return INITIAL_CODE.csharp.script } else if (language == 'nu') { return INITIAL_CODE.nu.script + } else if (language == 'java') { + return INITIAL_CODE.java.script + // KJQXZ } else if (language == 'bun' || language == 'bunnative') { if (kind == 'trigger') { return INITIAL_CODE.bun.trigger diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 95c0869e36..8c66851674 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -45,6 +45,9 @@ export function scriptLangToEditorLang( return 'csharp' } else if (lang == 'nu') { return 'nu' + } else if (lang == 'java') { + return 'java' + // KJQXZ } else if (lang == undefined) { return 'typescript' } else { @@ -125,7 +128,9 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['ansible', 'Ansible Playbook'], ['csharp', 'C#'], ['docker', 'Docker'], - ['nu', 'Nu'] + ['nu', 'Nu'], + ['java', 'Java'] + // KJQXZ ] export function processLangs(selected: string | undefined, langs: string[]): string[] { if (selected === 'nativets') { @@ -134,7 +139,8 @@ export function processLangs(selected: string | undefined, langs: string[]): str let ls = langs.filter((lang) => lang !== 'nativets') //those languages are newer and may not be in the saved list - let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu'] + let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java'] + // KJQXZ nl.forEach((lang) => { if (!ls.includes(lang)) { ls.push(lang) diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index b68e25a77b..ad7c93f53a 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -755,7 +755,8 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'dependency', 'nu'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)} +
Tag: {job.tag}