From 4317f065f507072d67276e3d7acf241da53183e3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 6 Jul 2023 21:19:07 +0200 Subject: [PATCH] feat: bun support (#1800) * feat: bun support * feat: bun support * add bun support * add bun support * add bun support * add bun --- Dockerfile | 2 + backend/src/main.rs | 7 +- backend/windmill-api/openapi.yaml | 8 +- .../nsjail/run.bash.config.proto | 5 +- .../nsjail/run.bun.config.proto | 121 ++++++ .../nsjail/run.go.config.proto | 5 +- backend/windmill-worker/src/global_cache.rs | 19 +- backend/windmill-worker/src/go_executor.rs | 1 + backend/windmill-worker/src/worker.rs | 149 ++++--- deno-client/mod.ts | 7 - deno-client/mysql.ts | 74 ---- deno-client/pg.ts | 57 --- .../src/lib/components/ScriptBuilder.svelte | 14 +- frontend/src/lib/script_helpers.ts | 5 +- typescript-client/.npmignore | 2 + typescript-client/build.sh | 3 + typescript-client/client.ts | 364 +++++++++++++++++ typescript-client/package-lock.json | 385 ++++++++++++++++++ typescript-client/package.json | 15 +- typescript-client/publish.sh | 2 + typescript-client/tsconfig.json | 5 +- 21 files changed, 1005 insertions(+), 245 deletions(-) create mode 100644 backend/windmill-worker/nsjail/run.bun.config.proto delete mode 100644 deno-client/mysql.ts delete mode 100644 deno-client/pg.ts create mode 100644 typescript-client/client.ts create mode 100644 typescript-client/package-lock.json diff --git a/Dockerfile b/Dockerfile index ee98da8f72..00ae1f4581 100644 --- a/Dockerfile +++ b/Dockerfile @@ -130,6 +130,8 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=nsjail /nsjail/nsjail /bin/nsjail COPY --from=denoland/deno:1.35.0 /usr/bin/deno /usr/bin/deno +COPY --from=oven/bun:0.6.13 /usr/bin/bun /usr/bin/bun + # docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image COPY --from=lukechannings/deno:v1.35.0 /usr/bin/deno /usr/bin/deno-arm diff --git a/backend/src/main.rs b/backend/src/main.rs index 666d1d6cbd..902503ae4e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -22,8 +22,9 @@ use tokio::{ use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT}; use windmill_common::{utils::rd_string, METRICS_ADDR}; use windmill_worker::{ - DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR, - HUB_TMP_CACHE_DIR, PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR, + BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, + GO_TMP_CACHE_DIR, HUB_CACHE_DIR, HUB_TMP_CACHE_DIR, PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, + TAR_PIP_TMP_CACHE_DIR, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -334,10 +335,12 @@ pub async fn run_workers) -> error::Resu "--filter", "+ deno/**", "--filter", + "+ bun/**", + "--filter", "+ go/**", "--filter", "+ tar/**", @@ -195,6 +197,8 @@ pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> { "--filter", "+ deno/**", "--filter", + "+ bun/**", + "--filter", "+ go/**", "--filter", "- *", @@ -226,6 +230,7 @@ pub async fn copy_cache_to_bucket_as_tar(bucket: &str) { &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"), "go", "deno", + "bun", ], ) .await @@ -277,7 +282,7 @@ pub async fn copy_cache_to_bucket_as_tar(bucket: &str) { pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) { use tokio::fs::metadata; - tracing::info!("Copying denogo cache from bucket {bucket} as tar"); + tracing::info!("Copying deno,go,bun cache from bucket {bucket} as tar"); let start: Instant = Instant::now(); @@ -295,7 +300,7 @@ pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) { ) .await { - tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e); + tracing::info!("Failed copying deno,go,bun tar from cache. Error: {:?}", e); return; } @@ -309,7 +314,7 @@ pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) { ) .await { - tracing::info!("Failed to untar denogo. Error: {:?}", e); + tracing::info!("Failed to untar denogobun tar. Error: {:?}", e); return; } @@ -321,12 +326,12 @@ pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) { if let Err(e) = tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await { - tracing::info!("Failed to remove denotar cache. Error: {:?}", e); + tracing::info!("Failed to remove denogobuntar cache. Error: {:?}", e); return; }; tracing::info!( - "Finished copying denogotar from bucket {bucket} as tar, took: {:?}s", + "Finished copying denogobuntar from bucket {bucket} as tar, took: {:?}s", start.elapsed().as_secs() ); } @@ -390,6 +395,8 @@ pub async fn copy_tmp_cache_to_cache() -> error::Result<()> { "--filter", "+ deno/**", "--filter", + "+ bun/**", + "--filter", "+ go/**", "--filter", "- *", @@ -480,6 +487,8 @@ pub async fn copy_cache_to_tmp_cache() -> error::Result<()> { "--filter", "+ deno/**", "--filter", + "+ bun/**", + "--filter", "+ go/**", "--filter", "- *", diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 41c23504b4..baf842ee0c 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -197,6 +197,7 @@ func Run(req Req) (interface{{}}, error){{ Command::new(NSJAIL_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(envs) .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b0746ef5e8..f213686524 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -136,10 +136,12 @@ pub const ROOT_TMP_CACHE_DIR: &str = "/tmp/windmill/tmpcache/"; pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go"); +pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); pub const HUB_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "hub"); pub const TAR_PIP_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "tar/pip"); pub const DENO_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "deno"); +pub const BUN_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "bun"); pub const GO_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "go"); pub const HUB_TMP_CACHE_DIR: &str = concatcp!(ROOT_TMP_CACHE_DIR, "hub"); @@ -148,6 +150,7 @@ const NUM_SECS_PING: u64 = 5; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); +const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto"); pub const DEFAULT_TIMEOUT: u64 = 900; @@ -187,6 +190,7 @@ lazy_static::lazy_static! { pub static ref HTTP_PROXY: Option = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok()); pub static ref HTTPS_PROXY: Option = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok()); pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); + pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| String::new()); @@ -1532,7 +1536,8 @@ mount {{ &inner_content, base_internal_url, worker_name, - envs + envs, + &shared_mount ) .await } @@ -1688,6 +1693,21 @@ fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap HashMap { + let mut deno_envs: HashMap = HashMap::from([ + (String::from("PATH"), PATH_ENV.clone()), + (String::from("DO_NOT_TRACK"), "1".to_string()), + (String::from("BASE_INTERNAL_URL"), base_internal_url.to_string()), + (String::from("BUN_INSTALL_CACHE_DIR"), BUN_CACHE_DIR.to_string()), + + ]); + + if let Some(ref s) = *NPM_CONFIG_REGISTRY { + deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone()); + } + return deno_envs; +} + #[tracing::instrument(level = "trace", skip_all)] async fn handle_deno_job( logs: &mut String, @@ -1811,7 +1831,7 @@ run().catch(async (e) => {{ //do not cache local dependencies let reload = format!("--reload={base_internal_url}"); - let child = async { + let child = { let script_path = format!("{job_dir}/wrapper.ts"); let import_map_path = format!("{job_dir}/import_map.json"); let mut args = Vec::with_capacity(12); @@ -1844,9 +1864,8 @@ run().catch(async (e) => {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn() - } - .await?; + .spawn()? + }; // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); // start = Instant::now(); handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id, "deno run").await?; @@ -1868,10 +1887,11 @@ async fn handle_bun_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + shared_mount: &str, ) -> error::Result { // let mut start = Instant::now(); - logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); + logs.push_str("\n\n--- BUN CODE EXECUTION ---\n"); let logs_to_set = logs.clone(); let id = job.id.clone(); @@ -1901,8 +1921,8 @@ async fn handle_bun_job( r#" import {{ main }} from "./main.ts"; -const args = await Deno.readTextFile("args.json") - .then(JSON.parse) + +const args = await Bun.file("args.json").json() .then(({{ {spread} }}) => [ {spread} ]) BigInt.prototype.toJSON = function () {{ @@ -1913,12 +1933,12 @@ BigInt.prototype.toJSON = function () {{ async function run() {{ let res: any = await main(...args); const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); - await Deno.writeTextFile("result.json", res_json); - Deno.exit(0); + await Bun.write("result.json", res_json); + process.exit(0); }} run().catch(async (e) => {{ - await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }})); - Deno.exit(1); + await Bun.write("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }})); + process.exit(1); }}); "#, ); @@ -1926,33 +1946,6 @@ run().catch(async (e) => {{ Ok(()) as error::Result<()> }; - let write_import_map_f = async { - let w_id = job.workspace_id.clone(); - let script_path_split = job.script_path().split("/"); - let script_path_parts_len = script_path_split.clone().count(); - let mut relative_mounts = "".to_string(); - for c in 0..script_path_parts_len { - relative_mounts += ",\n "; - relative_mounts += &format!("\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"", - (0..c).map(|_| "../").join(""), - &script_path_split.clone().take(script_path_parts_len - c - 1).join("/"), - if c == script_path_parts_len - 1 { "" } else { "/" }, - ); - } - let import_map = format!( - r#"{{ - "imports": {{ - "{base_internal_url}/api/w/{w_id}/scripts/raw/p/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", - "{base_internal_url}": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", - "/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/", - "./wrapper.ts": "./wrapper.ts", - "./main.ts": "./main.ts"{relative_mounts} - }} - }}"#, - ); - write_file(job_dir, "import_map.json", &import_map).await?; - Ok(()) as error::Result<()> - }; let reserved_variables_args_out_f = async { let client = client.get_authed().await; @@ -1961,68 +1954,68 @@ run().catch(async (e) => {{ Ok(()) as Result<()> }; let reserved_variables_f = async { - let mut vars = get_reserved_variables(job, &client.token, db).await?; - vars.insert("RUST_LOG".to_string(), "info".to_string()); + let vars = get_reserved_variables(job, &client.token, db).await?; Ok(vars) as Result> }; let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?; - Ok((reserved_variables, client.token)) as error::Result<(HashMap, String)> + Ok(reserved_variables) as error::Result> }; - let (_, (reserved_variables, token), _, _, _) = tokio::try_join!( + + let (_, reserved_variables, _, _) = tokio::try_join!( set_logs_f, reserved_variables_args_out_f, write_main_f, - write_wrapper_f, - write_import_map_f)?; + write_wrapper_f)?; + + let common_bun_proc_envs = get_common_bun_proc_envs(&base_internal_url); - let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url); //do not cache local dependencies - let reload = format!("--reload={base_internal_url}"); - let child = async { +let child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_BUN_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", BUN_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount), + ) + .await?; + + Command::new(NSJAIL_PATH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .envs(common_bun_proc_envs) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec!["--config", "run.config.proto", "--", &BUN_PATH, "run", "/tmp/bun/wrapper.ts", "--prefer-offline"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { let script_path = format!("{job_dir}/wrapper.ts"); - let import_map_path = format!("{job_dir}/import_map.json"); - let mut args = Vec::with_capacity(12); - args.push("run"); - args.push("--no-check"); - args.push("--import-map"); - args.push(&import_map_path); - args.push(&reload); - args.push("--unstable"); - if let Some(deno_flags) = DENO_FLAGS.as_ref() { - for flag in deno_flags { - args.push(flag); - } - } else if !*DISABLE_NSJAIL { - args.push("--allow-net"); - args.push("--allow-read=./"); - args.push("--allow-write=./"); - args.push("--allow-env"); - } else { - args.push("-A"); - } + let mut args = vec!["run", &script_path, "--prefer-offline"]; args.push(&script_path); - Command::new(DENO_PATH.as_str()) + Command::new(&*BUN_PATH) .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) - .envs(common_deno_proc_envs) - .env("DENO_DIR", DENO_CACHE_DIR) + .envs(common_bun_proc_envs) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn() - } - .await?; + .spawn()? + }; + // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); // start = Instant::now(); - handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id, "deno run").await?; + handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id, "bun run").await?; // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); - if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await { - tracing::error!("failed to remove deno gen tmp cache dir: {}", e); - } read_result(job_dir).await } diff --git a/deno-client/mod.ts b/deno-client/mod.ts index fd00b2ff7d..e8537df42e 100644 --- a/deno-client/mod.ts +++ b/deno-client/mod.ts @@ -21,13 +21,6 @@ export { WorkspaceService, } from "./windmill-api/index.ts"; -// @ts-ignore: Otherwise BigInt is not supported for export -BigInt.prototype.toJSON = function () { - return this.toString(); -}; - -export { pgSql, pgClient } from "./pg.ts"; - export type Sql = string; export type Email = string; export type Base64 = string; diff --git a/deno-client/mysql.ts b/deno-client/mysql.ts deleted file mode 100644 index ae00092f8a..0000000000 --- a/deno-client/mysql.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { createConnection } from "https://deno.land/x/mysql2@v1.0.6/mod.ts"; -import { type Resource } from "./mod.ts"; - -/** - * Establish MySQL connection using MySQL client for Deno: - * https://deno.land/x/mysql2@v1.0.6/mod.ts - * - * IMPORTANT: make sure to close the connection with `.end()` - * - * @param db the MySQL resource to establish the connection for - * - * @returns MySQL database connection - * - * @example - * ```ts - * const conn = await mysqlClient(db); - * await conn.execute('CREATE TABLE IF NOT EXISTS pets (name varchar(255), kind varchar(255))'); - * await conn.execute('INSERT INTO pets VALUES (?, ?)', ['behemot','cat']); - * const [rows] = await conn.execute('SELECT * from pets'); - * conn.end(); - * console.log(rows); - * ``` - */ -export async function mysqlClient( - db: Resource<"mysql"> -) { - const conn = await createConnection(db); - return conn; -} - -/** - * Execute SQL query. For more info check: - * https://deno.land/x/mysql2@v1.0.6/mod.ts - * - * @param db the MySQL resource to run sql query for - * - * @returns array with two items: rows and fields - * - * @example - * ```ts - * const kind = 'cat'; - * const { rows } = await mySql(db)`SELECT * from pets WHERE kind = ${kind}`; - * console.log(rows); - * ``` - */ -export function mySql( - db: Resource<"mysql">, - asObjects = false -) { - return async function execute( - query: TemplateStringsArray, - ...args: unknown[], - ) { - const conn = await mysqlClient(db); - const adapter = getQueryAdapter(query, args); - const [rows, fields] = await conn.execute(...adapter); - conn.end(); - return { rows: asObjects ? rows : getRowsAdapter(rows), fields }; - } -} - -function getQueryAdapter(template: TemplateStringsArray, args: unknown[]) { - const text = template.reduce((curr, next) => { - return `${curr}?${next}`; - }); - return [text, args]; -} - -function getRowsAdapter(rows: object[] | object) { - if (!Array.isArray(rows)) { - return rows; - } - return rows.map((r) => Object.values(r)) -} diff --git a/deno-client/pg.ts b/deno-client/pg.ts deleted file mode 100644 index 2c45a33534..0000000000 --- a/deno-client/pg.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts" -import { type Resource } from "./mod.ts" - -/** - * deno-postgres client API is very flexible: - * https://deno.land/x/postgres@v0.17.0/mod.ts?s=QueryClient - * - * @param db the PostgreSQL resource to generate the client for - * - * @returns the client for the resource - * - * @example - * // Static query - * ```ts - * const { rows } = await pgClient(db).queryObject( - * "SELECT ID, NAME FROM CLIENTS" - * ); - * ``` - * - * // Prepared Statements - * ```ts - * const { rows } = await pgClient(db).queryObject`SELECT ID, NAME FROM CLIENTS WHERE ID = ${id}`; - * ``` - */ -export function pgClient( - db: Resource<"postgresql"> -) { - const databaseUrl = 'postgresql://' + db.user + ':' + db.password + '@' + db.host + ':' + db.port + '/' + db.dbname + '?sslmode=' + db.sslmode - return new Client(databaseUrl) -} - -/** - * deno-postgres client API is very flexible: - * https://deno.land/x/postgres@v0.17.0/mod.ts?s=QueryClient - * - * @param db the PostgreSQL resource to run sql query for - * - * @returns the rows corresponding to the returned objetcs - * - * @example - * // Prepared Statements - * ```ts - * const { rows } = await pgSql(db)`SELECT ID, NAME FROM CLIENTS WHERE ID = ${id}`; - * ``` - */ -export function pgSql( - db: Resource<"postgresql">, - asObjects = false -) { - return async function queryObject( - query: TemplateStringsArray, - ...args: unknown[] - ) { - const client = pgClient(db) - return asObjects ? await client.queryObject(query, ...args) : await client.queryArray(query, ...args) - } -} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 2a2e2d1fbc..df2c7159d5 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -67,7 +67,6 @@ if (SCRIPT_SHOW_BASH) { langs.push(['Bash', Script.language.BASH]) } - // langs.push(['Typescript (Bun experimental)', Script.language.BUN]) langs.push(['PostgreSQL', Script.language.POSTGRESQL]) langs.push(['REST', Script.language.NATIVETS]) const scriptKindOptions: { @@ -331,6 +330,19 @@ > Docker +