diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json b/backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json similarity index 87% rename from backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json rename to backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json index c2c6738167..8ec0677716 100644 --- a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json +++ b/backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1", + "query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option\", envs AS \"envs: Vec\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar, codebase LIKE '%.esm%' as is_esm FROM script WHERE hash = $1 LIMIT 1", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 6, "name": "use_tar", "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "is_esm", + "type_info": "Bool" } ], "parameters": { @@ -82,8 +87,9 @@ true, true, false, + null, null ] }, - "hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a" + "hash": "7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662" } diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index d5d3e654a9..01200f1674 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -409,7 +409,7 @@ async fn create_snapshot_script( if name == "script" { let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap(); let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar")); - + let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm")); let (new_hash, ntx, hdm) = create_script_internal( ns, w_id.clone(), @@ -419,8 +419,14 @@ async fn create_snapshot_script( webhook.clone(), ) .await?; - let nh = new_hash.to_string(); - script_hash = Some(if is_tar { format!("{nh}.tar") } else { nh }); + let mut nh = new_hash.to_string(); + if use_esm { + nh = format!("{nh}.esm"); + } + if is_tar { + nh = format!("{nh}.tar"); + } + script_hash = Some(nh); tx = Some(ntx); handle_deployment_metadata = hdm; } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index df0ef6842a..8fef4b5e3a 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -631,7 +631,8 @@ pub mod script { envs AS \"envs: Vec\", \ schema AS \"schema: String\", \ schema_validation AS \"schema_validation: bool\", \ - codebase LIKE '%.tar' as use_tar \ + codebase LIKE '%.tar' as use_tar, \ + codebase LIKE '%.esm%' as is_esm \ FROM script WHERE hash = $1 LIMIT 1", hash.0 ) @@ -647,12 +648,14 @@ pub mod script { language: r.language, envs: r.envs, codebase: if let Some(use_tar) = r.use_tar { - let sh = hash.to_string(); - if use_tar { - Some(format!("{sh}.tar")) - } else { - Some(sh) + let mut sh = hash.to_string(); + if r.is_esm.unwrap_or(false) { + sh = format!("{sh}.esm"); } + if use_tar { + sh = format!("{sh}.tar"); + } + Some(sh) } else { None }, diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 1c95cb297d..de1ed3289b 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -142,7 +142,7 @@ export async function findResourceFile(path: string) { if (validCandidates.length > 1) { throw new Error( "Found two resource files for the same resource" + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -181,11 +181,11 @@ export async function handleScriptMetadata( } export interface OutputFile { - path: string - contents: Uint8Array - hash: string + path: string; + contents: Uint8Array; + hash: string; /** "contents" as text (changes automatically with "contents") */ - readonly text: string + readonly text: string; } export async function handleFile( @@ -223,10 +223,9 @@ export async function handleFile( let outputFiles: OutputFile[] = []; if (codebase.customBundler) { log.info(`Using custom bundler ${codebase.customBundler} for ${path}`); - bundleContent = execSync( - codebase.customBundler + " " + path, - { maxBuffer: 1024 * 1024 * 50 } - ).toString(); + bundleContent = execSync(codebase.customBundler + " " + path, { + maxBuffer: 1024 * 1024 * 50, + }).toString(); log.info("Custom bundler executed for " + path); } else { const esbuild = await import("npm:esbuild"); @@ -243,7 +242,7 @@ export async function handleFile( inject: codebase.inject, define: codebase.define, loader: codebase.loader ?? { ".node": "file" }, - outdir: '/', + outdir: "/", platform: "node", packages: "bundle", target: format == "cjs" ? "node20.15.1" : "esnext", @@ -260,17 +259,18 @@ export async function handleFile( if (outputFiles.length > 1) { const archiveNpm = await import("npm:@ayonli/jsext/archive"); log.info( - `Found multiple output files for ${path}, creating a tarball... ${outputFiles.map((file) => file.path).join(", ")}` + `Found multiple output files for ${path}, creating a tarball... ${outputFiles + .map((file) => file.path) + .join(", ")}` ); forceTar = true; const startTime = performance.now(); const tarball = new archiveNpm.Tarball(); const mainPath = path.split(SEP).pop()?.split(".")[0] + ".js"; - const content = outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? ''; + const content = + outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? ""; log.info(`Main content: ${content.length}chars`); - tarball.append( - new File([content], "main.js", { type: "text/plain" }) - ); + tarball.append(new File([content], "main.js", { type: "text/plain" })); for (const file of outputFiles) { if (file.path == "/" + mainPath) { continue; @@ -318,20 +318,20 @@ export async function handleFile( let typed = opts?.skipScriptsMetadata ? undefined : ( - await parseMetadataFile( - remotePath, - opts - ? { - ...opts, - path, - workspaceRemote: workspace, - schemaOnly: codebase ? true : undefined, - globalDeps, - codebases - } - : undefined, - ) - )?.payload; + await parseMetadataFile( + remotePath, + opts + ? { + ...opts, + path, + workspaceRemote: workspace, + schemaOnly: codebase ? true : undefined, + globalDeps, + codebases, + } + : undefined + ) + )?.payload; const workspaceId = workspace.workspaceId; @@ -401,6 +401,7 @@ export async function handleFile( on_behalf_of_email: typed?.on_behalf_of_email, }; + // console.log(requestBodyCommon.codebase); // log.info(JSON.stringify(requestBodyCommon, null, 2)) // log.info(JSON.stringify(opts, null, 2)) if (remote) { @@ -418,19 +419,19 @@ export async function handleFile( deepEqual(typed.schema, remote.schema) && typed.tag == remote.tag && (typed.ws_error_handler_muted ?? false) == - remote.ws_error_handler_muted && + remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && typed.concurrency_time_window_s == - remote.concurrency_time_window_s && + remote.concurrency_time_window_s && typed.concurrent_limit == remote.concurrent_limit && Boolean(typed.restart_unless_cancelled) == - Boolean(remote.restart_unless_cancelled) && + Boolean(remote.restart_unless_cancelled) && Boolean(typed.visible_to_runner_only) == - Boolean(remote.visible_to_runner_only) && + Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && Boolean(typed.has_preprocessor) == - Boolean(remote.has_preprocessor) && + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore @@ -523,7 +524,8 @@ async function createScript( }); } catch (e: any) { throw Error( - `Script creation for ${body.path} with parent ${body.parent_hash + `Script creation for ${body.path} with parent ${ + body.parent_hash } was not successful: ${e.body ?? e.message} ` ); } @@ -549,7 +551,8 @@ async function createScript( }); if (req.status != 201) { throw Error( - `Script snapshot creation was not successful: ${req.status} - ${req.statusText + `Script snapshot creation was not successful: ${req.status} - ${ + req.statusText } - ${await req.text()} ` ); } @@ -561,8 +564,8 @@ export async function findContentFile(filePath: string) { const candidates = filePath.endsWith("script.json") ? exts.map((x) => filePath.replace(".script.json", x)) : filePath.endsWith("script.lock") - ? exts.map((x) => filePath.replace(".script.lock", x)) - : exts.map((x) => filePath.replace(".script.yaml", x)); + ? exts.map((x) => filePath.replace(".script.lock", x)) + : exts.map((x) => filePath.replace(".script.yaml", x)); const validCandidates = ( await Promise.all( @@ -581,7 +584,7 @@ export async function findContentFile(filePath: string) { if (validCandidates.length > 1) { throw new Error( "No content path given and more than one candidate found: " + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { diff --git a/cli/src/utils/codebase.ts b/cli/src/utils/codebase.ts index 3391f6f795..84424f87c3 100644 --- a/cli/src/utils/codebase.ts +++ b/cli/src/utils/codebase.ts @@ -2,34 +2,46 @@ import { Codebase, SyncOptions } from "../core/conf.ts"; import { log } from "../../deps.ts"; import { digestDir } from "./utils.ts"; -export type SyncCodebase = Codebase & { getDigest: (forceTar?: boolean) => Promise }; -export function listSyncCodebases( - options: SyncOptions -): SyncCodebase[] { +export type SyncCodebase = Codebase & { + getDigest: (forceTar?: boolean) => Promise; +}; +export function listSyncCodebases(options: SyncOptions): SyncCodebase[] { const res: SyncCodebase[] = []; const nb_codebase = options?.codebases?.length ?? 0; if (nb_codebase > 0) { - log.info(`Found ${nb_codebase} codebases: ${options?.codebases?.map((c) => c.relative_path).join(", ")}`); + log.info( + `Found ${nb_codebase} codebases: ${options?.codebases + ?.map((c) => c.relative_path) + .join(", ")}` + ); } for (const codebase of options?.codebases ?? []) { let _digest: string | undefined = undefined; let alreadyPrinted = false; - const getDigest: (forceTar?: boolean) => Promise = async (forceTar?: boolean) => { - if (_digest == undefined || forceTar) { + let hasAssets = false; + const getDigest: (forceTar?: boolean) => Promise = async ( + forceTar?: boolean + ) => { + if (_digest == undefined) { _digest = await digestDir( codebase.relative_path, JSON.stringify(codebase) ); - if (forceTar || (Array.isArray(codebase.assets) && codebase.assets.length > 0)) { - _digest += ".tar"; + if (codebase.format == "esm") { + _digest += ".esm"; } if (!alreadyPrinted) { alreadyPrinted = true; log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); } + hasAssets = + Array.isArray(codebase.assets) && codebase.assets.length > 0; + } + if (forceTar || hasAssets) { + return _digest + ".tar"; + } else { return _digest; } - return _digest; }; res.push({ ...codebase, getDigest }); }