From acd6a019c3120dd31f7bd69d64a1f2be974aa1a4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 5 Jul 2023 10:01:00 +0200 Subject: [PATCH] select + drawer fix + explicit hash --- backend/windmill-api/openapi.yaml | 10 +- backend/windmill-api/src/workspaces.rs | 1 + backend/windmill-common/src/scripts.rs | 2 + backend/windmill-queue/src/jobs.rs | 1 + backend/windmill-worker/src/go_executor.rs | 4 +- .../windmill-worker/src/python_executor.rs | 2 +- backend/windmill-worker/src/worker.rs | 189 +++++++++++++++++- .../apps/components/inputs/AppSelect.svelte | 4 +- .../apps/components/layout/AppModal.svelte | 2 +- frontend/src/lib/utils.ts | 6 +- .../(root)/(logged)/apps/add/+page.svelte | 2 +- .../(root)/(logged)/run/[...run]/+page.svelte | 6 +- 12 files changed, 211 insertions(+), 18 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 445413ad8b..8b63572ef5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5550,7 +5550,7 @@ components: type: string language: type: string - enum: [python3, deno, go, bash, postgresql, nativets] + enum: [python3, deno, go, bash, postgresql, nativets, bun] kind: type: string enum: [script, failure, trigger, command, approval] @@ -5605,7 +5605,7 @@ components: type: string language: type: string - enum: [python3, deno, go, bash, postgresql, nativets] + enum: [python3, deno, go, bash, postgresql, nativets, bun] kind: type: string enum: [script, failure, trigger, command, approval] @@ -5765,7 +5765,7 @@ components: type: boolean language: type: string - enum: [python3, deno, go, bash, postgresql, nativets] + enum: [python3, deno, go, bash, postgresql, nativets, bun] email: type: string visible_to_owner: @@ -5855,7 +5855,7 @@ components: type: boolean language: type: string - enum: [python3, deno, go, bash, postgresql, nativets] + enum: [python3, deno, go, bash, postgresql, nativets, bun] is_skipped: type: boolean email: @@ -6283,7 +6283,7 @@ components: $ref: "#/components/schemas/ScriptArgs" language: type: string - enum: [python3, deno, go, bash, postgresql, nativets] + enum: [python3, deno, go, bash, postgresql, nativets, bun] tag: type: string kind: diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 75e75066a5..353b00748c 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -1337,6 +1337,7 @@ async fn tarball_workspace( ScriptLang::Bash => "sh", ScriptLang::Postgresql => "pg.sql", ScriptLang::Nativets => "fetch.ts", + ScriptLang::Bun => "bun.ts", }; archive .write_to_archive(&script.content, &format!("{}.{}", script.path, ext)) diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 836d154c28..7989aba5ad 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -31,11 +31,13 @@ pub enum ScriptLang { Go, Bash, Postgresql, + Bun, } impl ScriptLang { pub fn as_str(&self) -> &'static str { match self { + ScriptLang::Bun => "bun", ScriptLang::Nativets => "nativets", ScriptLang::Deno => "deno", ScriptLang::Python3 => "python3", diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index b93c6011ae..738c748b83 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -71,6 +71,7 @@ lazy_static::lazy_static! { "go".to_string(), "bash".to_string(), "nativets".to_string(), + "bun".to_string(), "postgresql".to_string(), "dependency".to_string(), "flow".to_string(), diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index a35bc187f9..41c23504b4 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -305,7 +305,7 @@ pub async fn install_go_dependencies( .fetch_optional(db) .await? { - logs.push_str(&format!("\nfound cached resolution")); + logs.push_str(&format!("\nfound cached resolution: {}", hash)); gen_go_mod(code, job_dir, &cached).await?; skip_tidy = true; new_lockfile = false; @@ -333,7 +333,7 @@ pub async fn install_go_dependencies( &format!("go {mod_command}"), ) .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + .map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?; if (!new_lockfile || has_sum) && non_dep_job { return Ok("".to_string()); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index dc751913fd..a0a956c0ef 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -103,7 +103,7 @@ pub async fn pip_compile( .fetch_optional(db) .await? { - logs.push_str(&format!("\nfound cached resolution:\n{cached}")); + logs.push_str(&format!("\nfound cached resolution: {req_hash}")); return Ok(cached); } let file = "requirements.in"; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d4e3865a07..b0746ef5e8 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -352,7 +352,8 @@ pub async fn run_worker = all_langs.clone().into_iter().map(|x| (x.clone(), prometheus::register_histogram!( prometheus::HistogramOpts::new( @@ -1521,6 +1522,20 @@ mount {{ ) .await } + Some(ScriptLang::Bun) => { + handle_bun_job( + logs, + job, + db, + client, + job_dir, + &inner_content, + base_internal_url, + worker_name, + envs + ) + .await + } Some(ScriptLang::Go) => { handle_go_job( logs, @@ -1842,6 +1857,175 @@ run().catch(async (e) => {{ read_result(job_dir).await } +#[tracing::instrument(level = "trace", skip_all)] +async fn handle_bun_job( + logs: &mut String, + job: &QueuedJob, + db: &sqlx::Pool, + client: &AuthedClientBackgroundTask, + job_dir: &str, + inner_content: &String, + base_internal_url: &str, + worker_name: &str, + envs: HashMap, +) -> error::Result { + + // let mut start = Instant::now(); + logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); + + let logs_to_set = logs.clone(); + let id = job.id.clone(); + let db2 = db.clone(); + + let set_logs_f = async { + set_logs(&logs_to_set, &id, &db2).await; + Ok(()) as error::Result<()> + }; + + let write_main_f = write_file(job_dir, "main.ts", inner_content); + + let write_wrapper_f = async { + // let mut start = Instant::now(); + let args = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args; + let dates = args.iter().enumerate().filter_map(|(i, x)| if matches!(x.typ, Typ::Datetime) { + Some(i) + } else { + None + }).map(|x| { + return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined") + }).join("\n"); + + let spread = args.into_iter().map(|x| x.name).join(","); + // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); + let wrapper_content: String = format!( + r#" +import {{ main }} from "./main.ts"; + +const args = await Deno.readTextFile("args.json") + .then(JSON.parse) + .then(({{ {spread} }}) => [ {spread} ]) + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +{dates} +async function run() {{ + let res: any = await main(...args); + const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await Deno.writeTextFile("result.json", res_json); + Deno.exit(0); +}} +run().catch(async (e) => {{ + await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }})); + Deno.exit(1); +}}); + "#, + ); + write_file(job_dir, "wrapper.ts", &wrapper_content).await?; + Ok(()) as error::Result<()> + }; + + let write_import_map_f = async { + let w_id = job.workspace_id.clone(); + let script_path_split = job.script_path().split("/"); + let script_path_parts_len = script_path_split.clone().count(); + let mut relative_mounts = "".to_string(); + for c in 0..script_path_parts_len { + relative_mounts += ",\n "; + relative_mounts += &format!("\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"", + (0..c).map(|_| "../").join(""), + &script_path_split.clone().take(script_path_parts_len - c - 1).join("/"), + if c == script_path_parts_len - 1 { "" } else { "/" }, + ); + } + let import_map = format!( + r#"{{ + "imports": {{ + "{base_internal_url}/api/w/{w_id}/scripts/raw/p/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", + "{base_internal_url}": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", + "/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", + "./wrapper.ts": "./wrapper.ts", + "./main.ts": "./main.ts"{relative_mounts} + }} + }}"#, + ); + write_file(job_dir, "import_map.json", &import_map).await?; + Ok(()) as error::Result<()> + }; + + let reserved_variables_args_out_f = async { + let client = client.get_authed().await; + let args_and_out_f = async { + create_args_and_out_file(&client, job, job_dir).await?; + Ok(()) as Result<()> + }; + let reserved_variables_f = async { + let mut vars = get_reserved_variables(job, &client.token, db).await?; + vars.insert("RUST_LOG".to_string(), "info".to_string()); + Ok(vars) as Result> + }; + let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; + Ok((reserved_variables, client.token)) as error::Result<(HashMap, String)> + }; + + let (_, (reserved_variables, token), _, _, _) = tokio::try_join!( + set_logs_f, + reserved_variables_args_out_f, + write_main_f, + write_wrapper_f, + write_import_map_f)?; + + let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url); + + //do not cache local dependencies + let reload = format!("--reload={base_internal_url}"); + let child = async { + let script_path = format!("{job_dir}/wrapper.ts"); + let import_map_path = format!("{job_dir}/import_map.json"); + let mut args = Vec::with_capacity(12); + args.push("run"); + args.push("--no-check"); + args.push("--import-map"); + args.push(&import_map_path); + args.push(&reload); + args.push("--unstable"); + if let Some(deno_flags) = DENO_FLAGS.as_ref() { + for flag in deno_flags { + args.push(flag); + } + } else if !*DISABLE_NSJAIL { + args.push("--allow-net"); + args.push("--allow-read=./"); + args.push("--allow-write=./"); + args.push("--allow-env"); + } else { + args.push("-A"); + } + args.push(&script_path); + Command::new(DENO_PATH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .envs(common_deno_proc_envs) + .env("DENO_DIR", DENO_CACHE_DIR) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + } + .await?; + // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); + // start = Instant::now(); + handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id, "deno run").await?; + // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); + if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await { + tracing::error!("failed to remove deno gen tmp cache dir: {}", e); + } + read_result(job_dir).await +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn create_args_and_out_file( client: &AuthedClient, @@ -2074,6 +2258,9 @@ async fn capture_dependency_job( Ok(String::new()) // generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, timeout).await }, + ScriptLang::Bun => { + Ok(String::new()) + }, ScriptLang::Postgresql => Ok("".to_owned()), ScriptLang::Bash => Ok("".to_owned()), ScriptLang::Nativets => Ok("".to_owned()), diff --git a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte index b6f25529f7..36a94cfce4 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte @@ -192,9 +192,9 @@ } }} > -
{#if resolvedConfig.create}{item.created ? 'Add new: ' : ''}{/if}{item.label} -
+ diff --git a/frontend/src/lib/components/apps/components/layout/AppModal.svelte b/frontend/src/lib/components/apps/components/layout/AppModal.svelte index 933ab02daf..b5b9259d64 100644 --- a/frontend/src/lib/components/apps/components/layout/AppModal.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppModal.svelte @@ -108,7 +108,7 @@ 'm-24 max-h-[80%] bg-white overflow-y-auto rounded-lg relative', css?.popup?.class )} - use:clickOutside + use:clickOutside={false} on:click_outside={() => { if ($mode !== 'dnd') { closeDrawer() diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index b0cd9aa177..2c42c0d794 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -112,18 +112,18 @@ export function validatePassword(password: string): boolean { return re.test(password) } -export function clickOutside(node: Node): { destroy(): void } { +export function clickOutside(node: Node, capture?: boolean): { destroy(): void } { const handleClick = (event: MouseEvent) => { if (node && !node.contains(event.target) && !event.defaultPrevented) { node.dispatchEvent(new CustomEvent('click_outside', { detail: event })) } } - document.addEventListener('click', handleClick, true) + document.addEventListener('click', handleClick, capture ?? true) return { destroy() { - document.removeEventListener('click', handleClick, true) + document.removeEventListener('click', handleClick, capture ?? true) } } } diff --git a/frontend/src/routes/(root)/(logged)/apps/add/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/add/+page.svelte index 1145f8e18d..9ecab09bd7 100644 --- a/frontend/src/routes/(root)/(logged)/apps/add/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/add/+page.svelte @@ -84,7 +84,7 @@ sendUserToast('App loaded from Hub') goto('?', { replaceState: true }) } else if (!templatePath && !hubId && state) { - sendUserToast('App restored from draft', false, [ + sendUserToast('App restored from browser stored autosave', false, [ { label: 'Start from blank', callback: () => { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 85cb46d133..54d98c08b7 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -333,7 +333,7 @@ Result Logs {#if job?.job_kind == 'dependencies'} - Dependencies + Code {:else if job?.job_kind == 'preview'} Code {/if} @@ -354,7 +354,9 @@ {:else if viewTab == 'code'} {#if job && 'raw_code' in job && job.raw_code} - +
+ +
{:else if job} No code is available {:else}