From f94fed5ee7df6d759ec285cb74686bc300ac92cf Mon Sep 17 00:00:00 2001 From: wendrul Date: Thu, 5 Dec 2024 16:28:57 +0100 Subject: [PATCH] Add csharp stuff and merge after the rewrite * Add csharp, boilerplate + minimal execution * Add initial parser * Frontend + wasm export of the tree sitter parser * Arg spread and use cache --- backend/Cargo.lock | 41 +++ backend/Cargo.toml | 4 + ...20241029132207_add-csharp-support.down.sql | 1 + .../20241029132207_add-csharp-support.up.sql | 3 + .../parsers/windmill-parser-csharp/Cargo.toml | 20 ++ .../parsers/windmill-parser-csharp/src/lib.rs | 142 ++++++++ .../windmill-parser-csharp/src/wasm_libc.rs | 208 +++++++++++ .../parsers/windmill-parser-wasm/Cargo.toml | 2 + .../windmill-parser-wasm/build-pkgs-mac.sh | 6 + .../windmill-parser-wasm/build-pkgs.sh | 5 + .../windmill-parser-wasm/publish-pkgs.sh | 3 + .../parsers/windmill-parser-wasm/src/lib.rs | 6 + .../wasm-sysroot/assert.h | 4 + .../windmill-parser-wasm/wasm-sysroot/ctype.h | 3 + .../wasm-sysroot/inttypes.h | 3 + .../wasm-sysroot/stdbool.h | 5 + .../wasm-sysroot/stdint.h | 19 + .../windmill-parser-wasm/wasm-sysroot/stdio.h | 19 + .../wasm-sysroot/stdlib.h | 12 + .../wasm-sysroot/string.h | 7 + .../windmill-parser-wasm/wasm-sysroot/time.h | 5 + .../wasm-sysroot/unistd.h | 3 + .../wasm-sysroot/wctype.h | 7 + backend/src/main.rs | 4 +- backend/tests/worker.rs | 41 +++ backend/windmill-api/openapi.yaml | 6 + backend/windmill-api/src/scripts.rs | 1 + backend/windmill-api/src/workspaces.rs | 1 + backend/windmill-common/src/scripts.rs | 2 + backend/windmill-common/src/worker.rs | 1 + backend/windmill-worker/Cargo.toml | 1 + .../windmill-worker/src/ansible_executor.rs | 23 +- backend/windmill-worker/src/common.rs | 18 + .../windmill-worker/src/csharp_executor.rs | 339 ++++++++++++++++++ backend/windmill-worker/src/lib.rs | 1 + backend/windmill-worker/src/php_executor.rs | 25 +- backend/windmill-worker/src/rust_executor.rs | 25 +- backend/windmill-worker/src/worker.rs | 21 ++ .../windmill-worker/src/worker_lockfiles.rs | 36 ++ frontend/package-lock.json | 4 + frontend/src/lib/components/Editor.svelte | 1 + frontend/src/lib/components/EditorBar.svelte | 9 +- .../src/lib/components/HighlightCode.svelte | 3 + .../src/lib/components/WorkerGroup.svelte | 3 +- .../common/languageIcons/LanguageIcon.svelte | 7 +- .../lib/components/icons/CSharpIcon.svelte | 6 + frontend/src/lib/editorUtils.ts | 2 + frontend/src/lib/infer.ts | 8 + frontend/src/lib/script_helpers.ts | 20 ++ frontend/src/lib/scripts.ts | 3 + .../(root)/(logged)/run/[...run]/+page.svelte | 2 +- 51 files changed, 1068 insertions(+), 73 deletions(-) create mode 100644 backend/migrations/20241029132207_add-csharp-support.down.sql create mode 100644 backend/migrations/20241029132207_add-csharp-support.up.sql create mode 100644 backend/parsers/windmill-parser-csharp/Cargo.toml create mode 100644 backend/parsers/windmill-parser-csharp/src/lib.rs create mode 100644 backend/parsers/windmill-parser-csharp/src/wasm_libc.rs create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/assert.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/ctype.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/inttypes.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/stdbool.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/stdint.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/stdio.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/stdlib.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/time.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/unistd.h create mode 100644 backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h create mode 100644 backend/windmill-worker/src/csharp_executor.rs create mode 100644 frontend/src/lib/components/icons/CSharpIcon.svelte diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 54758013bc..5bc149f29a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9692,6 +9692,34 @@ dependencies = [ "tracing-serde 0.2.0", ] +[[package]] +name = "tree-sitter" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0203df02a3b6dd63575cc1d6e609edc2181c9a11867a271b25cfd2abff3ec5ca" +dependencies = [ + "cc", + "regex", + "regex-syntax 0.8.5", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-c-sharp" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c0f6d2209a3cd6d0bb9d2934715da15a15710d3c09c7c1ecd4c9804c3ecd10" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ddffe35a0e5eeeadf13ff7350af564c6e73993a24db62caee1822b185c2600" + [[package]] name = "triomphe" version = "0.1.14" @@ -10737,6 +10765,17 @@ dependencies = [ "windmill-parser", ] +[[package]] +name = "windmill-parser-csharp" +version = "1.435.1" +dependencies = [ + "anyhow", + "tree-sitter", + "tree-sitter-c-sharp", + "wasm-bindgen", + "windmill-parser", +] + [[package]] name = "windmill-parser-go" version = "1.435.1" @@ -10859,6 +10898,7 @@ dependencies = [ "wasm-bindgen-test", "windmill-parser", "windmill-parser-bash", + "windmill-parser-csharp", "windmill-parser-go", "windmill-parser-graphql", "windmill-parser-php", @@ -10986,6 +11026,7 @@ dependencies = [ "windmill-git-sync", "windmill-parser", "windmill-parser-bash", + "windmill-parser-csharp", "windmill-parser-go", "windmill-parser-graphql", "windmill-parser-php", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6fdf37998c..af76d0fea9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -21,6 +21,7 @@ members = [ "./parsers/windmill-parser-wasm", "./parsers/windmill-parser-go", "./parsers/windmill-parser-rust", + "./parsers/windmill-parser-csharp", "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", @@ -128,6 +129,7 @@ windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" } windmill-parser-go = { path = "./parsers/windmill-parser-go" } windmill-parser-rust = { path = "./parsers/windmill-parser-rust" } windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" } +windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } @@ -299,3 +301,5 @@ quote = "1.0.36" regex-lite = "0.1.6" yaml-rust = "0.4.5" tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } +tree-sitter = {version = "0.23.0", features = []} +tree-sitter-c-sharp = "0.23.0" diff --git a/backend/migrations/20241029132207_add-csharp-support.down.sql b/backend/migrations/20241029132207_add-csharp-support.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20241029132207_add-csharp-support.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20241029132207_add-csharp-support.up.sql b/backend/migrations/20241029132207_add-csharp-support.up.sql new file mode 100644 index 0000000000..128cae9e77 --- /dev/null +++ b/backend/migrations/20241029132207_add-csharp-support.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'csharp'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["csharp"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible"]}'::jsonb AND NOT config->'worker_tags' @> '"csharp"'::jsonb; diff --git a/backend/parsers/windmill-parser-csharp/Cargo.toml b/backend/parsers/windmill-parser-csharp/Cargo.toml new file mode 100644 index 0000000000..6e2c14e0ca --- /dev/null +++ b/backend/parsers/windmill-parser-csharp/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "windmill-parser-csharp" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_csharp" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +tree-sitter.workspace = true +tree-sitter-c-sharp.workspace = true +anyhow.workspace = true +wasm-bindgen.workspace = true +# convert_case.workspace = true +# lazy_static.workspace = true +# regex.workspace = true + diff --git a/backend/parsers/windmill-parser-csharp/src/lib.rs b/backend/parsers/windmill-parser-csharp/src/lib.rs new file mode 100644 index 0000000000..5c42bb3624 --- /dev/null +++ b/backend/parsers/windmill-parser-csharp/src/lib.rs @@ -0,0 +1,142 @@ +#![feature(c_variadic)] + +#[cfg(target_arch = "wasm32")] +pub mod wasm_libc; + +use anyhow::anyhow; +use tree_sitter::Node; +use windmill_parser::Arg; +use windmill_parser::MainArgSignature; +use windmill_parser::Typ; + +pub fn parse_csharp_signature(code: &str) -> anyhow::Result { + let mut parser = tree_sitter::Parser::new(); + let language = tree_sitter_c_sharp::LANGUAGE; + parser + .set_language(&language.into()) + .map_err(|e| anyhow!("Error setting c# as language: {e}"))?; + + // Parse code + let tree = parser.parse(code, None).expect("Failed to parse code"); + let root_node = tree.root_node(); + + // Traverse the AST to find the Main method signature + let main_sig = find_main_signature(root_node, code); + let no_main_func = Some(main_sig.is_none()); + + let mut args = vec![]; + if let Some(sig) = main_sig { + // for (i, c) in sig.children(&mut sig.walk()).enumerate() { + // println!(" {:?} - {:?}", c, sig.field_name_for_child((i) as u32)); + // } + if let Some(param_list) = sig.child_by_field_name("parameters") { + for c in param_list.children(&mut param_list.walk()) { + if c.kind() == "parameter" { + let (otyp, typ, name) = parse_csharp_typ(c, code); + args.push(Arg { + name, + otyp, + typ, + default: None, + has_default: false, + oidx: None, + }); + for (i, w) in c.children(&mut c.walk()).enumerate() { + let s = w.utf8_text(code.as_bytes()); + println!( + " {:?} - {:?} - {:?}", + w, + c.field_name_for_child((i) as u32), + s + ); + } + } + } + } else { + println!("No one with parameter_list"); + } + } + + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + has_preprocessor: None, + no_main_func, + }) +} + +fn parse_csharp_typ<'a>(param_node: Node<'a>, code: &str) -> (Option, Typ, String) { + let name = param_node + .child_by_field_name("name") + .and_then(|n| n.utf8_text(code.as_bytes()).ok()) + .unwrap_or(""); + let otyp_node = param_node.child_by_field_name("type"); + let otyp = otyp_node + .and_then(|n| n.utf8_text(code.as_bytes()).ok()) + .map(|s| s.to_string()); + + let typ = Typ::Str(None); + + (otyp, typ, name.to_string()) +} + +// Function to find the Main method's signature +fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> Option> { + let mut cursor = root_node.walk(); + for x in root_node.children(&mut cursor) { + if x.kind() == "class_declaration" { + for c in x.children(&mut x.walk()) { + if c.kind() == "declaration_list" { + for w in c.children(&mut c.walk()) { + if w.kind() == "method_declaration" { + for child in w.children(&mut w.walk()) { + if child + .utf8_text(code.as_bytes()) + .map(|name| name == "Main") + .unwrap_or(false) + { + return Some(w); + } + } + } + } + } + } + } + } + return None; +} + +#[cfg(test)] +mod test { + + use super::*; + #[test] + fn test_parse_csharp_sig() { + let code = r#" +using System; + +class LilProgram +{ + + public static string Main(string myString = "World", int myInt) + { + Console.Writeline("Hello!!"); + return "yeah"; + } + +}"#; + let ret = parse_csharp_signature(code).unwrap(); + + assert_eq!(ret.args.len(), 2); + + assert_eq!(ret.args[0].name, "myString"); + assert_eq!(ret.args[0].otyp, Some("string".to_string())); + assert_eq!(ret.args[0].typ, Typ::Str(None)); + + assert_eq!(ret.args[1].name, "myInt"); + assert_eq!(ret.args[1].otyp, Some("int".to_string())); + assert_eq!(ret.args[1].typ, Typ::Int); + } +} diff --git a/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs b/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs new file mode 100644 index 0000000000..bb703fb131 --- /dev/null +++ b/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs @@ -0,0 +1,208 @@ +use std::{ + alloc::{self, Layout}, + ffi::{c_char, c_int, c_void}, + mem::align_of, + ptr, +}; +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; +use wasm_bindgen::prelude::*; + +/* -------------------------------- stdlib.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn abort() { + panic!("Aborted from C"); +} + +macro_rules! console_log { + ($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) }) +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn log(a: &str); +} + +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size); + let buf = alloc::alloc(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + if count == 0 || size == 0 { + return ptr::null_mut(); + } + + let (layout, offset_to_data) = layout_for_size_prepended(size * count); + let buf = alloc::alloc_zeroed(layout); + store_layout(buf, layout, offset_to_data) +} + +#[no_mangle] +pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void { + if buf.is_null() { + malloc(new_size) + } else if new_size == 0 { + free(buf); + ptr::null_mut() + } else { + let (old_buf, old_layout) = retrieve_layout(buf); + let (new_layout, offset_to_data) = layout_for_size_prepended(new_size); + let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size()); + store_layout(new_buf, new_layout, offset_to_data) + } +} + +#[no_mangle] +pub unsafe extern "C" fn free(buf: *mut c_void) { + if buf.is_null() { + return; + } + let (buf, layout) = retrieve_layout(buf); + alloc::dealloc(buf, layout); +} + + +// In all these allocations, we store the layout before the data for later retrieval. +// This is because we need to know the layout when deallocating the memory. +// Here are some helper methods for that: + +/// Given a pointer to the data, retrieve the layout and the pointer to the layout. +unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) { + let (_, layout_offset) = Layout::new::() + .extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap()) + .unwrap(); + + let buf = (buf as *mut u8).offset(-(layout_offset as isize)); + let layout = *(buf as *mut Layout); + + (buf, layout) +} + +/// Calculate a layout for a given size with space for storing a layout at the start. +/// Returns the layout and the offset to the data. +fn layout_for_size_prepended(size: usize) -> (Layout, usize) { + Layout::new::() + .extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap()) + .unwrap() +} + +/// Store a layout in the pointer, returning a pointer to where the data should be stored. +unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void { + *(buf as *mut Layout) = layout; + (buf as *mut u8).offset(offset_to_data as isize) as *mut c_void +} + +/* -------------------------------- string.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int { + let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n); + let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n); + + for (a, b) in s1.iter().zip(s2.iter()) { + if *a != *b || *a == 0 { + return (*a as i32) - (*b as i32); + } + } + + 0 +} + +/* -------------------------------- wctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn iswspace(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_whitespace()) +} + +#[no_mangle] +pub unsafe extern "C" fn iswalnum(c: c_int) -> bool { + char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric()) +} + +/* --------------------------------- time.h --------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn clock() -> u64 { + panic!("clock is not supported"); +} + +/* --------------------------------- ctype.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn isprint(c: c_int) -> bool { + c >= 32 && c <= 126 +} + +/* --------------------------------- stdio.h -------------------------------- */ + +#[no_mangle] +pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int { + panic!("fprintf is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int { + panic!("fputs is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int { + panic!("fputc is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void { + panic!("fdopen is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int { + panic!("fclose is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn fwrite( + _ptr: *const c_void, + _size: usize, + _nmemb: usize, + _stream: *mut c_void, +) -> usize { + panic!("fwrite is not supported"); +} + +#[no_mangle] +pub unsafe extern "C" fn vsnprintf( + _buf: *mut c_char, + _size: usize, + _format: *const c_char, + _args: ... +) -> c_int { + panic!("vsnprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) { + panic!("asdasd"); +} + +// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... ); +#[no_mangle] +pub extern "C" fn snprintf() { + panic!("snprintf is not supported"); +} + +#[no_mangle] +pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) { + panic!("oh no"); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 2513ece67b..c78a9da69f 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -26,6 +26,7 @@ php-parser = [ "dep:windmill-parser-php"] rust-parser = [ "dep:windmill-parser-rust"] graphql-parser = [ "dep:windmill-parser-graphql"] ansible-parser = [ "dep:windmill-parser-yaml"] +csharp-parser = [ "dep:windmill-parser-csharp"] [dependencies] anyhow.workspace = true @@ -39,6 +40,7 @@ windmill-parser-php = { workspace = true, optional = true } windmill-parser-graphql = { workspace = true, optional = true } windmill-parser-rust = { workspace = true, optional = true } windmill-parser-yaml = { workspace = true, optional = true } +windmill-parser-csharp = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true getrandom = { workspace = true, features = ["js"] } diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh index 7803fe22f0..dcd5a78275 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh @@ -48,3 +48,9 @@ OUT_DIR="pkg-yaml" wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json + +# C# (needs some more stuff to compile C tree sitter into wasm) +# TODO: hasn't been tested on mac, might need fixing +OUT_DIR="pkg-csharp" +CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "csharp-parser" +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-csharp"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/build-pkgs.sh b/backend/parsers/windmill-parser-wasm/build-pkgs.sh index 66dafe3ccc..ec7a1f5ee4 100755 --- a/backend/parsers/windmill-parser-wasm/build-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs.sh @@ -48,3 +48,8 @@ OUT_DIR="pkg-yaml" wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json + +# C# (needs some more stuff to compile C tree sitter into wasm) +OUT_DIR="pkg-csharp" +CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "csharp-parser" +sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-csharp"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index c2f2c0d7ad..a4af8bb684 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -24,3 +24,6 @@ popd pushd "pkg-yaml" && npm publish ${args} popd + +pushd "pkg-csharp" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index a5dd03ad91..50ebf232ac 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -135,3 +135,9 @@ pub fn parse_rust(code: &str) -> String { pub fn parse_ansible(code: &str) -> String { wrap_sig(windmill_parser_yaml::parse_ansible_sig(code)) } + +#[cfg(feature = "csharp-parser")] +#[wasm_bindgen] +pub fn parse_csharp(code: &str) -> String { + wrap_sig(windmill_parser_csharp::parse_csharp_signature(code)) +} diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/assert.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/assert.h new file mode 100644 index 0000000000..8a17e8dc6f --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/assert.h @@ -0,0 +1,4 @@ +#pragma once + +#define assert(ignore) ((void)0) +#define static_assert(cnd, msg) assert(cnd && msg) diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/ctype.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/ctype.h new file mode 100644 index 0000000000..14175419ab --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/ctype.h @@ -0,0 +1,3 @@ +#pragma once + +int isprint(int c); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/inttypes.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/inttypes.h new file mode 100644 index 0000000000..cfc0873a81 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/inttypes.h @@ -0,0 +1,3 @@ +#pragma once + +#define PRId32 "d" diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdbool.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdbool.h new file mode 100644 index 0000000000..97287ba6e9 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdbool.h @@ -0,0 +1,5 @@ +#pragma once + +#define bool _Bool +#define true 1 +#define false 0 diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdint.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdint.h new file mode 100644 index 0000000000..d575a081b4 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdint.h @@ -0,0 +1,19 @@ +#pragma once + +typedef signed char int8_t; +typedef short int16_t; +typedef long int32_t; +typedef long long int64_t; + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned long uint32_t; +typedef unsigned long long uint64_t; + +typedef unsigned long size_t; + +typedef unsigned int uintptr_t; + +#define UINT8_MAX 0xff +#define UINT16_MAX 0xffff +#define UINT32_MAX 0xffffffff diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdio.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdio.h new file mode 100644 index 0000000000..6e19a95aa3 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdio.h @@ -0,0 +1,19 @@ +#pragma once + +// just some filler type +#define FILE void + +#define stdin NULL +#define stdout NULL +#define stderr NULL + +int fprintf(FILE *__restrict__, const char *__restrict__, ...); +int fputs(const char *__restrict, FILE *__restrict); +int fputc(int, FILE *); +FILE *fdopen(int, const char *); +int fclose(FILE *); + +int vsnprintf(char *s, unsigned long n, const char *format, ...); + +#define sprintf(str, ...) 0 +#define snprintf(str, len, ...) 0 diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdlib.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdlib.h new file mode 100644 index 0000000000..8c8df2a8fd --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/stdlib.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +#define NULL ((void*)0) + +void* malloc(size_t size); +void* calloc(size_t nmemb, size_t size); +void free(void* ptr); +void* realloc(void* ptr, size_t size); + +void abort(void); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h new file mode 100644 index 0000000000..c0687e9c88 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/string.h @@ -0,0 +1,7 @@ +#pragma once + +void *memcpy(void *dest, const void *src, unsigned long n); +void *memmove(void *dest, const void *src, unsigned long n); +void *memset(void *s, int c, unsigned long n); +int memcmp(const void *ptr1, const void *ptr2, unsigned long n); +int strncmp(const char *s1, const char *s2, unsigned long n); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/time.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/time.h new file mode 100644 index 0000000000..3a1583bdaf --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/time.h @@ -0,0 +1,5 @@ +#pragma once + +typedef unsigned long clock_t; +#define CLOCKS_PER_SEC ((clock_t)1000000) +clock_t clock(void); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/unistd.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/unistd.h new file mode 100644 index 0000000000..b3727dd3ef --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/unistd.h @@ -0,0 +1,3 @@ +#pragma once + +int dup(int); diff --git a/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h b/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h new file mode 100644 index 0000000000..517a954be5 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/wasm-sysroot/wctype.h @@ -0,0 +1,7 @@ +#pragma once + +typedef __WCHAR_TYPE__ wchar_t; +typedef __WINT_TYPE__ wint_t; + +int iswspace(wchar_t ch); +int iswalnum(wint_t _wc); diff --git a/backend/src/main.rs b/backend/src/main.rs index d403ad212a..3d509cedaf 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -68,8 +68,9 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, TMP_LOGS_DIR, + PY311_CACHE_DIR, CSHARP_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, + RUST_CACHE_DIR, CSHARP_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1001,6 +1002,7 @@ pub async fn run_workers( GO_CACHE_DIR, GO_BIN_CACHE_DIR, RUST_CACHE_DIR, + CSHARP_CACHE_DIR, HUB_CACHE_DIR, POWERSHELL_CACHE_DIR, ] { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index a229450651..1498c0168a 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1794,6 +1794,47 @@ fn main(world: String) -> Result { assert_eq!(result, serde_json::json!("Hello Hyrule!")); } +#[sqlx::test(fixtures("base"))] +async fn test_csharp_job(db: Pool) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + + let content = r#" +using System; + +class Script +{ + public static string moin(string world) + { + Console.WriteLine($"Hello {world}"); + return $"Hello {world}"; + } +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::CSharp, + custom_concurrency_key: None, + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + })) + .arg("world", json!("Arakis")) + .run_until_complete(&db, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("Hello Arakis")); +} + #[sqlx::test(fixtures("base"))] async fn test_bash_job(db: Pool) { initialize_tracing().await; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 089f330cbb..c30c186f1e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10671,6 +10671,7 @@ components: php, rust, ansible, + csharp, ] kind: type: string @@ -10774,6 +10775,7 @@ components: php, rust, ansible, + csharp, ] kind: type: string @@ -10996,6 +10998,7 @@ components: php, rust, ansible, + csharp, ] email: type: string @@ -11116,6 +11119,7 @@ components: php, rust, ansible, + csharp, ] is_skipped: type: boolean @@ -11669,6 +11673,7 @@ components: php, rust, ansible, + csharp, ] tag: type: string @@ -13285,6 +13290,7 @@ components: php, rust, ansible, + csharp, ] required: - raw_code diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index ba21c688e4..090bff24aa 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -583,6 +583,7 @@ async fn create_script_internal<'c>( || ns.language == ScriptLang::Deno || ns.language == ScriptLang::Rust || ns.language == ScriptLang::Ansible + || ns.language == ScriptLang::CSharp || ns.language == ScriptLang::Php) { Some(String::new()) diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index bba06b238b..f9e6f70b97 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2364,6 +2364,7 @@ async fn tarball_workspace( ScriptLang::Php => "php", ScriptLang::Rust => "rs", ScriptLang::Ansible => "playbook.yml", + ScriptLang::CSharp => "cs", }; archive .write_to_archive(&script.content, &format!("{}.{}", script.path, ext)) diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 1a11b5a701..8176edaa0c 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -45,6 +45,7 @@ pub enum ScriptLang { Php, Rust, Ansible, + CSharp, } impl ScriptLang { @@ -67,6 +68,7 @@ impl ScriptLang { ScriptLang::Php => "php", ScriptLang::Rust => "rust", ScriptLang::Ansible => "ansible", + ScriptLang::CSharp => "csharp", } } } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9a4064ff1d..42c02e9a7f 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -45,6 +45,7 @@ lazy_static::lazy_static! { "php".to_string(), "rust".to_string(), "ansible".to_string(), + "csharp".to_string(), "dependency".to_string(), "flow".to_string(), "other".to_string() diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index cb64757a7f..a8f065a4d2 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -28,6 +28,7 @@ windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-go.workspace = true windmill-parser-rust.workspace = true +windmill-parser-csharp.workspace = true windmill-parser-py.workspace = true windmill-parser-yaml.workspace = true windmill-parser-py-imports.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index dcc288f108..9bb67a1a6c 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -29,8 +29,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ bash_executor::BIN_BASH, common::{ - get_reserved_variables, read_and_check_result, start_child_process, transform_json, - OccupancyMetrics, + check_executor_binary_exists, get_reserved_variables, read_and_check_result, start_child_process, transform_json, OccupancyMetrics }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, @@ -185,24 +184,6 @@ async fn install_galaxy_collections( Ok(()) } -#[cfg(not(feature = "enterprise"))] -fn check_ansible_exists() -> Result<(), error::Error> { - if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); - return Err(error::Error::NotFound(msg)); - } - Ok(()) -} - -#[cfg(feature = "enterprise")] -fn check_ansible_exists() -> Result<(), error::Error> { - if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); - return Err(error::Error::NotFound(msg)); - } - Ok(()) -} - pub async fn handle_ansible_job( requirements_o: Option, job_dir: &str, @@ -219,7 +200,7 @@ pub async fn handle_ansible_job( envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - check_ansible_exists()?; + check_executor_binary_exists("ansible-playbook", ANSIBLE_PLAYBOOK_PATH.as_str(), "ansible")?; let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?; append_logs(&job.id, &job.workspace_id, logs, db).await; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 31d8ecc4dd..b12406e512 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -28,6 +28,7 @@ use windmill_common::{ use anyhow::{anyhow, Result}; +use std::path::Path; use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, @@ -55,6 +56,23 @@ pub async fn build_args_map<'a>( return Ok(None); } +pub fn check_executor_binary_exists( + executor: &str, + executor_path: &str, + language: &str, +) -> Result<(), Error> { + if !Path::new(executor_path).exists() { + #[cfg(feature = "enterprise")] + let msg = format!("Couldn't find {executor} at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run {language} jobs.", executor_path); + + #[cfg(not(feature = "enterprise"))] + let msg = format!("Couldn't find {executor} at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run {language} jobs.", executor_path); + return Err(Error::NotFound(msg)); + } + + Ok(()) +} + pub async fn build_args_values( job: &QueuedJob, client: &AuthedClientBackgroundTask, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs new file mode 100644 index 0000000000..863e21bd91 --- /dev/null +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -0,0 +1,339 @@ +use anyhow::anyhow; +use serde_json::value::RawValue; +use std::{collections::HashMap, path::Path, process::Stdio}; +use uuid::Uuid; +use windmill_parser_rust::parse_rust_deps_into_manifest; + +use itertools::Itertools; +use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use windmill_common::{ + error::{self, Error}, + jobs::QueuedJob, + utils::calculate_hash, + worker::{save_cache, write_file}, +}; +use windmill_queue::{append_logs, CanceledBy}; + +use crate::{ + common::{ + check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, + }, + handle_child::handle_child, + AuthedClientBackgroundTask, CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, + HOME_ENV, NSJAIL_PATH, PATH_ENV, RUST_CACHE_DIR, TZ_ENV, +}; + +#[cfg(windows)] +use crate::SYSTEM_ROOT; + +lazy_static::lazy_static! { + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); +} + +const CSHARP_OBJECT_STORE_PREFIX: &str = "csharpbin/"; + +fn gen_cs_proj(code: &str, job_dir: &str) -> anyhow::Result<()> { + write_file( + job_dir, + "Main.csproj", + r#" + + + Exe + net7.0 + enable + enable + WindmillScriptCSharpInternal.Wrapper + + + +"#, + )?; + + write_file(job_dir, "Script.cs", code)?; + + let sig = windmill_parser_csharp::parse_csharp_signature(code)?; + let spread = &sig + .args + .clone() + .into_iter() + .map(|x| format!("parsedArgs.{}", &x.name)) + .join(", "); + let args_class_body = &sig + .args + .into_iter() + .map(|x| { + Ok(format!( + " public {} {} {{ get; set; }}", + x.otyp.ok_or(anyhow!("Type not found for argument {}", x.name))?, + &x.name, + )) + }) + .collect::, anyhow::Error>>()? + .join("\n"); + + write_file( + job_dir, + "Wrapper.cs", + &format!( + r#"using System; +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; + +namespace WindmillScriptCSharpInternal {{ + + struct Args {{ +{args_class_body} + }} + + class Wrapper + {{ + static void Main(string[] args) + {{ + using FileStream fs = File.OpenRead("args.json"); + Args parsedArgs = JsonSerializer.Deserialize(fs); + + var result = Script.Main({spread}); + + var jsonResult = JsonSerializer.Serialize(result); + + File.WriteAllText("result.json", jsonResult); + }} + }} +}} +"#, + ), + )?; + + Ok(()) +} + +async fn build_cs_proj( + job_id: &Uuid, + mem_peak: &mut i32, + canceled_by: &mut Option, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + w_id: &str, + base_internal_url: &str, + hash: &str, + occupancy_metrics: &mut OccupancyMetrics, +) -> error::Result { + let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str()); + + build_cs_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + // .env("CARGO_HOME", CARGO_HOME.as_str()) + // .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .args(vec![ + "publish", + "--configuration", + "Release", + "-r", + "linux-x64", + "-o", + job_dir, + "--no-self-contained", + "-p:PublishSingleFile=true", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + build_cs_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_cs_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } + + let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str()).await?; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + build_cs_process, + false, + worker_name, + w_id, + "dotnet publish", + None, + false, + &mut Some(occupancy_metrics), + ) + .await?; + append_logs(job_id, w_id, "\n\n", db).await; + + for entry in std::fs::read_dir(job_dir)? { + let entry = entry?; + let path = entry.path(); + + // Print file or directory name + if let Some(name) = path.file_name() { + // println!("{}", name.to_string_lossy()); + println!("{path:?}"); + } + } + + let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + + match save_cache( + &bin_path, + &format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"), + &format!("{job_dir}/Main"), + ) + .await + { + Err(e) => { + let em = format!("could not save {job_dir}/Main to C# cache: {e:?}",); + tracing::error!(em); + Ok(em) + } + Ok(logs) => Ok(logs), + } +} + +pub async fn handle_csharp_job( + mem_peak: &mut i32, + canceled_by: &mut Option, + job: &QueuedJob, + db: &sqlx::Pool, + client: &AuthedClientBackgroundTask, + inner_content: &str, + job_dir: &str, + requirements_o: Option, + shared_mount: &str, + base_internal_url: &str, + worker_name: &str, + envs: HashMap, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result, Error> { + check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; + + + + let hash = calculate_hash(&format!( + "{}{}", + inner_content, + requirements_o + .as_ref() + .map(|x| x.to_string()) + .unwrap_or_default() + )); + let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR); + let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); + + let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await; + + let cache_logs = if cache { + let target = format!("{job_dir}/Main"); + + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { + Error::ExecutionErr(format!( + "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + )) + })?; + + cache_logs + } else { + let logs1 = format!("{cache_logs}\n\n--- DOTNET BUILD ---\n"); + append_logs(&job.id, &job.workspace_id, logs1, db).await; + + gen_cs_proj(inner_content, job_dir)?; + + build_cs_proj( + &job.id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + &job.workspace_id, + base_internal_url, + &hash, + occupancy_metrics, + ) + .await? + }; + + create_args_and_out_file(client, job, job_dir, db).await?; + + let logs2 = format!("{cache_logs}\n\n--- C# CODE EXECUTION ---\n"); + append_logs(&job.id, &job.workspace_id, logs2, db).await; + + let client = &client.get_authed().await; + let reserved_variables = get_reserved_variables(job, &client.token, db).await?; + + let child = if !*DISABLE_NSJAIL { + todo!(); + // let _ = write_file( + // job_dir, + // "run.config.proto", + // &NSJAIL_CONFIG_RUN_RUST_CONTENT + // .replace("{JOB_DIR}", job_dir) + // .replace("{CACHE_DIR}", RUST_CACHE_DIR) + // .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + // .replace("{SHARED_MOUNT}", shared_mount), + // )?; + // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + // nsjail_cmd + // .current_dir(job_dir) + // .env_clear() + // .envs(envs) + // .envs(reserved_variables) + // .env("PATH", PATH_ENV.as_str()) + // .env("TZ", TZ_ENV.as_str()) + // .env("BASE_INTERNAL_URL", base_internal_url) + // .args(vec!["--config", "run.config.proto", "--", "/tmp/main"]) + // .stdout(Stdio::piped()) + // .stderr(Stdio::piped()); + // start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { + let compiled_executable_name = "./Main"; + let mut run_csharp = Command::new(compiled_executable_name); + run_csharp + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + start_child_process(run_csharp, compiled_executable_name).await? + }; + + handle_child( + &job.id, + db, + mem_peak, + canceled_by, + child, + !*DISABLE_NSJAIL, + worker_name, + &job.workspace_id, + "csharp run", + job.timeout, + false, + &mut Some(occupancy_metrics), + ) + .await?; + read_result(job_dir).await +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 680f3ca178..3a2efd75d8 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -30,6 +30,7 @@ mod worker; mod worker_flow; mod worker_lockfiles; mod job_logger_ee; +mod csharp_executor; pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 194e621fe1..345d8e37b7 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -15,8 +15,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - create_args_and_out_file, get_main_override, get_reserved_variables, read_result, - start_child_process, OccupancyMetrics, + check_executor_binary_exists, create_args_and_out_file, get_main_override, get_reserved_variables, read_result, start_child_process, OccupancyMetrics }, handle_child::handle_child, AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, @@ -73,7 +72,7 @@ pub async fn composer_install( lock: Option, occupancy_metrics: &mut OccupancyMetrics, ) -> Result { - check_php_exists()?; + check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?; write_file(job_dir, "composer.json", &requirements)?; @@ -131,24 +130,6 @@ $args->{arg_name} = new {rt_name}($args->{arg_name});" ) } -#[cfg(not(feature = "enterprise"))] -fn check_php_exists() -> error::Result<()> { - if !Path::new(PHP_PATH.as_str()).exists() { - let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run php jobs.", PHP_PATH.as_str()); - return Err(error::Error::NotFound(msg)); - } - Ok(()) -} - -#[cfg(feature = "enterprise")] -fn check_php_exists() -> error::Result<()> { - if !Path::new(PHP_PATH.as_str()).exists() { - let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run php jobs.", PHP_PATH.as_str()); - return Err(error::Error::NotFound(msg)); - } - Ok(()) -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_php_job( requirements_o: Option, @@ -165,7 +146,7 @@ pub async fn handle_php_job( shared_mount: &str, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - check_php_exists()?; + check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?; let (composer_json, composer_lock) = match requirements_o { Some(reqs_and_lock) if !reqs_and_lock.is_empty() => { diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index c6f258d6ac..7032823cb2 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -15,8 +15,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, @@ -132,7 +131,7 @@ pub async fn generate_cargo_lockfile( w_id: &str, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result { - check_cargo_exists()?; + check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; gen_cargo_crate(code, job_dir)?; @@ -271,24 +270,6 @@ pub fn compute_rust_hash(code: &str, requirements_o: Option<&String>) -> String )) } -#[cfg(not(feature = "enterprise"))] -fn check_cargo_exists() -> Result<(), Error> { - if !Path::new(CARGO_PATH.as_str()).exists() { - let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); - return Err(Error::NotFound(msg)); - } - Ok(()) -} - -#[cfg(feature = "enterprise")] -fn check_cargo_exists() -> Result<(), Error> { - if !Path::new(CARGO_PATH.as_str()).exists() { - let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); - return Err(Error::NotFound(msg)); - } - Ok(()) -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_rust_job( mem_peak: &mut i32, @@ -305,7 +286,7 @@ pub async fn handle_rust_job( envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, ) -> Result, Error> { - check_cargo_exists()?; + check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; let hash = compute_rust_hash(inner_content, requirements_o.as_ref()); let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 47f2fb393f..36430b5495 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -94,6 +94,7 @@ use crate::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, OccupancyMetrics, }, + csharp_executor::handle_csharp_job, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, @@ -263,6 +264,7 @@ pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); +pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); pub const BUN_DEPSTAR_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "buntar"); @@ -365,6 +367,7 @@ lazy_static::lazy_static! { pub static ref POWERSHELL_PATH: String = std::env::var("POWERSHELL_PATH").unwrap_or_else(|_| "/usr/bin/pwsh".to_string()); pub static ref PHP_PATH: String = std::env::var("PHP_PATH").unwrap_or_else(|_| "/usr/bin/php".to_string()); pub static ref COMPOSER_PATH: String = std::env::var("COMPOSER_PATH").unwrap_or_else(|_| "/usr/bin/composer".to_string()); + pub static ref DOTNET_PATH: String = std::env::var("DOTNET_PATH").unwrap_or_else(|_| "/usr/bin/dotnet".to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); @@ -2667,6 +2670,24 @@ mount {{ ) .await } + Some(ScriptLang::CSharp) => { + handle_csharp_job( + mem_peak, + canceled_by, + job, + db, + client, + &inner_content, + job_dir, + requirements_o, + &shared_mount, + base_internal_url, + worker_name, + envs, + occupancy_metrics, + ) + .await + } _ => panic!("unreachable, language is not supported: {language:#?}"), }; tracing::info!( diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 71834a9f04..73a07394ca 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1764,6 +1764,42 @@ async fn capture_dependency_job( .await?; Ok(lockfile) } + ScriptLang::CSharp => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for C#".to_string(), + )); + } + Ok("".to_string()) + + // let lockfile = generate_cargo_lockfile( + // job_id, + // job_raw_code, + // mem_peak, + // canceled_by, + // job_dir, + // db, + // worker_name, + // w_id, + // occupancy_metrics, + // ) + // .await?; + // + // build_rust_crate( + // job_id, + // mem_peak, + // canceled_by, + // job_dir, + // db, + // worker_name, + // w_id, + // base_internal_url, + // &compute_rust_hash(&job_raw_code, Some(&lockfile)), + // occupancy_metrics, + // ) + // .await?; + // Ok(lockfile) + } ScriptLang::Postgresql => Ok("".to_owned()), ScriptLang::Mysql => Ok("".to_owned()), ScriptLang::Bigquery => Ok("".to_owned()), diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 13b3a8612a..adbe47c219 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -142,6 +142,10 @@ "svelte": "^4.0.0" } }, + "../backend/parsers/windmill-parser-wasm/pkg-csharp": { + "name": "windmill-parser-wasm-csharp", + "version": "1.411.1" + }, "../svelte-dnd-action": { "extraneous": true }, diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index ba23b76fdc..27818a6c54 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -191,6 +191,7 @@ | 'javascript' | 'rust' | 'yaml' + | 'csharp' export let code: string = '' export let cmdEnterAction: (() => void) | undefined = undefined export let formatAction: (() => void) | undefined = undefined diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index e3d09ee007..a99419f90f 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -91,7 +91,8 @@ 'bunnative', 'nativets', 'php', - 'rust' + 'rust', + 'csharp' ].includes(lang ?? '') $: showVarPicker = [ 'python3', @@ -103,7 +104,8 @@ 'bunnative', 'nativets', 'php', - 'rust' + 'rust', + 'csharp' ].includes(lang ?? '') $: showResourcePicker = [ 'python3', @@ -115,7 +117,8 @@ 'bunnative', 'nativets', 'php', - 'rust' + 'rust', + 'csharp' ].includes(lang ?? '') $: showResourceTypePicker = ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) || diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 9f9d525625..80857ef0b0 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -10,6 +10,7 @@ import powershell from 'svelte-highlight/languages/powershell' import php from 'svelte-highlight/languages/php' import rust from 'svelte-highlight/languages/rust' + import csharp from 'svelte-highlight/languages/csharp' import yaml from 'svelte-highlight/languages/yaml' import type { Script } from '$lib/gen' import { Button } from './common' @@ -55,6 +56,8 @@ return php case 'rust': return rust + case 'csharp': + return csharp case 'ansible': return yaml; default: diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 100468fb26..5916860d4f 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -121,7 +121,8 @@ 'bun', 'php', 'rust', - 'ansible' + 'ansible', + 'csharp' ] const nativeTags = [ 'nativets', diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 881d8ba72f..9bf4241891 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -18,6 +18,7 @@ import PHPIcon from '$lib/components/icons/PHPIcon.svelte' import RustIcon from '$lib/components/icons/RustIcon.svelte' import AnsibleIcon from '$lib/components/icons/AnsibleIcon.svelte' + import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte' export let lang: | SupportedLanguage @@ -50,7 +51,8 @@ bun: 'TypeScript', php: 'PHP', rust: 'Rust', - ansible: 'Ansible Playbook' + ansible: 'Ansible Playbook', + csharp: 'C sharpo' } const langToComponent: Record< @@ -78,7 +80,8 @@ graphql: GraphqlIcon, php: PHPIcon, rust: RustIcon, - ansible: AnsibleIcon + ansible: AnsibleIcon, + csharp: CSharpIcon } let subIconScale = width === 30 ? 0.6 : 0.8 diff --git a/frontend/src/lib/components/icons/CSharpIcon.svelte b/frontend/src/lib/components/icons/CSharpIcon.svelte new file mode 100644 index 0000000000..ed0c0742fd --- /dev/null +++ b/frontend/src/lib/components/icons/CSharpIcon.svelte @@ -0,0 +1,6 @@ + + +c# diff --git a/frontend/src/lib/editorUtils.ts b/frontend/src/lib/editorUtils.ts index 03ce7420ca..33c54c4a0f 100644 --- a/frontend/src/lib/editorUtils.ts +++ b/frontend/src/lib/editorUtils.ts @@ -80,6 +80,8 @@ export function langToExt(lang: string): string { return 'css' case 'ansible': return 'yml' + case 'csharp': + return 'cs' default: return 'unknown' } diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 0bb53957f5..604ee9802c 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -21,6 +21,7 @@ import initGoParser, { parse_go } from 'windmill-parser-wasm-go' import initPhpParser, { parse_php } from 'windmill-parser-wasm-php' import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust' import initYamlParser, { parse_ansible } from 'windmill-parser-wasm-yaml' +import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp' import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url' import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url' @@ -29,6 +30,7 @@ import wasmUrlGo from 'windmill-parser-wasm-go/windmill_parser_wasm_bg.wasm?url' import wasmUrlPhp from 'windmill-parser-wasm-php/windmill_parser_wasm_bg.wasm?url' import wasmUrlRust from 'windmill-parser-wasm-rust/windmill_parser_wasm_bg.wasm?url' import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm?url' +import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.wasm?url' import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from './inferArgSig.js' @@ -60,6 +62,9 @@ async function initWasmGo() { async function initWasmYaml() { await initYamlParser(wasmUrlYaml) } +async function initWasmCSharp() { + await initCSharpParser(wasmUrlCSharp) +} export async function inferArgs( language: SupportedLanguage | 'bunnative' | undefined, @@ -161,6 +166,9 @@ export async function inferArgs( } else if (language == 'ansible') { await initWasmYaml() inferedSchema = JSON.parse(parse_ansible(code)) + } else if (language == 'csharp') { + await initWasmCSharp() + inferedSchema = JSON.parse(parse_csharp(code)) } else { return null } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 9f78127b45..9f50d352d3 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -348,6 +348,21 @@ fn main(who_to_greet: String, numbers: Vec) -> anyhow::Result { } ` +const CSHARP_INIT_CODE = ` +using System; + +class LilProgram +{ + + public static string Main(string myString = "World", int myInt) + { + Console.Writeline("Hello!!"); + return "yeah"; + } + +} +` + const FETCH_INIT_CODE = `export async function main( url: string | undefined, method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'OPTIONS' = 'GET', @@ -767,6 +782,9 @@ export const INITIAL_CODE = { ansible: { script: ANSIBLE_PLAYBOOK_INIT_CODE }, + csharp: { + script: CSHARP_INIT_CODE + }, docker: { script: DOCKER_INIT_CODE }, @@ -871,6 +889,8 @@ export function initialCode( return INITIAL_CODE.rust.script } else if (language == 'ansible') { return INITIAL_CODE.ansible.script + } else if (language == 'csharp') { + return INITIAL_CODE.csharp.script } else if (language == 'bun' || language == 'bunnative') { if (kind == 'trigger') { return INITIAL_CODE.bun.trigger diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 36c6a439da..f741b71775 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -39,6 +39,8 @@ export function scriptLangToEditorLang( return 'graphql' } else if (lang == 'ansible') { return 'yaml' + } else if (lang == 'csharp') { + return 'csharp' } else if (lang == undefined) { return 'typescript' } else { @@ -116,6 +118,7 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['php', 'PHP'], ['rust', 'Rust'], ['ansible', 'Ansible Playbook'], + ['csharp', 'C#'], ['docker', 'Docker'] ] export function processLangs(selected: string | undefined, langs: string[]): string[] { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 1669b17720..c310a18f42 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -730,7 +730,7 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'dependency'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'dependency'].includes(job.tag)}
Tag: {job.tag}